Create sysinv-utils for generating host-overrides

This commits creates a new utility command /usr/bin/sysinv-utils
that can be used by ansible playbooks to generate the host overrides
file that will be used for upgrades and backup/restore.

This commits also enhanced the following CLI commands to include a
--column and --format options for better CLI outputs data manipulation.
	host-show
	host-list
	service-parameter-list
        application-show

These two new options are optional. The default behavior is not changed.

Story: 2006590
Task: 36892
Change-Id: I36b8ea411a4eb74d6e1a2194982025905e9299e5
Signed-off-by: Kristine Bujold <kristine.bujold@windriver.com>

sysinv/cgts-client/centos/build_srpm.data

@@ -1,2 +1,2 @@
 SRC_DIR="cgts-client"
-TIS_PATCH_VER=72
+TIS_PATCH_VER=73

sysinv/cgts-client/cgts-client/cgtsclient/common/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2017 Wind River, Inc
+# Copyright 2013-2019 Wind River, Inc
 # Copyright 2012 OpenStack LLC.
 # All Rights Reserved.
 #
@@ -22,6 +22,7 @@ except Exception:
     is_remote = True
 
 import argparse
+from collections import OrderedDict
 import copy
 import dateutil
 import math
@@ -32,6 +33,7 @@ import six
 import sys
 import textwrap
 import uuid
+import yaml
 
 from prettytable import ALL
 from prettytable import FRAME
@@ -364,11 +366,25 @@ def parse_date(string_data):
 
 
 def print_list(objs, fields, field_labels, formatters={}, sortby=0,
-               reversesort=False, no_wrap_fields=[], printer=default_printer):
+               reversesort=False, no_wrap_fields=[], printer=default_printer,
-    # print_list() is the same as print_long_list() with paging turned off
+               output_format=None):
-    return print_long_list(objs, fields, field_labels, formatters=formatters, sortby=sortby,
+
-                           reversesort=reversesort, no_wrap_fields=no_wrap_fields,
+    if output_format == 'yaml' or output_format == 'value':
-                           no_paging=True, printer=printer)
+        my_dict_list = []
+        for o in objs:
+            my_dict_list.append(dict((k, getattr(o, k)) for k in fields))
+
+        if output_format == 'yaml':
+            print(yaml.safe_dump(my_dict_list, default_flow_style=False))
+
+        elif output_format == 'value':
+            for _dict in my_dict_list:
+                print_dict_value(_dict)
+
+    else:
+        return print_long_list(objs, fields, field_labels, formatters=formatters, sortby=sortby,
+                               reversesort=reversesort, no_wrap_fields=no_wrap_fields,
+                               no_paging=True, printer=printer)
 
 
 def _build_row_from_object(fields, formatters, o):
@@ -445,6 +461,24 @@ def print_long_list(objs, fields, field_labels, formatters={}, sortby=0, reverse
     pt.done()
 
 
+def print_dict_with_format(data, wrap=0, output_format=None):
+    if output_format == 'yaml':
+        print(yaml.safe_dump(data, default_flow_style=False))
+
+    elif output_format == 'value':
+        print_dict_value(data)
+
+    else:
+        ordereddata = OrderedDict(sorted(data.items(), key=lambda t: t[0]))
+        print_dict(ordereddata, wrap=wrap)
+
+
+def print_dict_value(d):
+    # Print values on a single line separated by spaces
+    # e.g. 'available ntp'
+    print (' '.join(map(str, d.values())))
+
+
 def print_dict(d, dict_property="Property", wrap=0):
     pt = prettytable.PrettyTable([dict_property, 'Value'],
                                  caching=False, print_empty=False)

sysinv/cgts-client/cgts-client/cgtsclient/tests/test_shell.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013-2014 Wind River Systems, Inc.
+# Copyright (c) 2013-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -78,12 +78,13 @@ class ShellTest(utils.BaseTestCase):
 
     def test_help_on_subcommand(self):
         required = [
-            '.*?^usage: system host-show <hostname or id>'
+            '.*?^usage: system host-show \[--column COLUMN\] \[--format {table,yaml,value}\]'
+            '.*?<hostname or id>'
             '',
             ".*?^Show host attributes.",
             '',
             ".*?^Positional arguments:",
-            ".*?^  <hostname or id>  Name or ID of host",
+            ".*?<hostname or id>      Name or ID of host",
         ]
         argstrings = [
             'help host-show',

sysinv/cgts-client/cgts-client/cgtsclient/v1/app_shell.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -9,13 +9,17 @@ import re
 
 from cgtsclient.common import utils
 from cgtsclient import exc
-from collections import OrderedDict
 
 
-def _print_application_show(app):
+def _print_application_show(app, columns=None, output_format=None):
-    ordereddata = OrderedDict(sorted(app.to_dict().items(),
+
-                                     key=lambda t: t[0]))
+    if columns:
-    utils.print_dict(ordereddata, wrap=72)
+        data_dict = dict((k, v) for (k, v) in vars(app).items()
+                         if k in columns and not (v is None))
+    else:
+        data_dict = app.to_dict()
+
+    utils.print_dict_with_format(data_dict, wrap=72, output_format=output_format)
 
 
 def _print_reminder_msg(app_name):
@@ -73,11 +77,18 @@ def do_application_list(cc, args):
 
 @utils.arg('name', metavar='<app name>',
            help="Name of the application")
+@utils.arg('--column',
+           action='append',
+           default=[],
+           help="Specify the column(s) to include, can be repeated")
+@utils.arg('--format',
+           choices=['table', 'yaml', 'value'],
+           help="specify the output format, defaults to table")
 def do_application_show(cc, args):
     """Show application details"""
     try:
         app = cc.app.get(args.name)
-        _print_application_show(app)
+        _print_application_show(app, args.column, args.format)
     except exc.HTTPNotFound:
         raise exc.CommandError('application not found: %s' % args.name)
 

sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py

@@ -9,7 +9,6 @@
 # All Rights Reserved.
 #
 
-from collections import OrderedDict
 import datetime
 import os
 
@@ -25,24 +24,29 @@ from cgtsclient.v1 import istor as istor_utils
 from six.moves import input
 
 
-def _print_ihost_show(ihost):
+def _print_ihost_show(ihost, columns=None, output_format=None):
-    fields = ['id', 'uuid', 'personality', 'hostname', 'invprovision',
+    optional_fields = []
-              'administrative', 'operational', 'availability', 'task',
+    if columns:
-              'action', 'mgmt_mac', 'mgmt_ip', 'serialid',
+        fields = columns
-              'capabilities', 'bm_type', 'bm_username', 'bm_ip',
+    else:
-              'config_applied', 'config_target', 'config_status',
+        fields = ['id', 'uuid', 'personality', 'hostname', 'invprovision',
-              'location', 'uptime', 'reserved', 'created_at', 'updated_at',
+                  'administrative', 'operational', 'availability', 'task',
-              'boot_device', 'rootfs_device', 'install_output', 'console',
+                  'action', 'mgmt_mac', 'mgmt_ip', 'serialid',
-              'tboot', 'vim_progress_status', 'software_load', 'install_state',
+                  'capabilities', 'bm_type', 'bm_username', 'bm_ip',
-              'install_state_info', 'inv_state', 'clock_synchronization']
+                  'config_applied', 'config_target', 'config_status',
-    optional_fields = ['vsc_controllers', 'ttys_dcd']
+                  'location', 'uptime', 'reserved', 'created_at', 'updated_at',
-    if ihost.subfunctions != ihost.personality:
+                  'boot_device', 'rootfs_device', 'install_output', 'console',
-        fields.append('subfunctions')
+                  'tboot', 'vim_progress_status', 'software_load',
-        if 'controller' in ihost.subfunctions:
+                  'install_state', 'install_state_info', 'inv_state',
-            fields.append('subfunction_oper')
+                  'clock_synchronization']
-            fields.append('subfunction_avail')
+        optional_fields = ['vsc_controllers', 'ttys_dcd']
-    if ihost.peers:
+        if ihost.subfunctions != ihost.personality:
-        fields.append('peers')
+            fields.append('subfunctions')
+            if 'controller' in ihost.subfunctions:
+                fields.append('subfunction_oper')
+                fields.append('subfunction_avail')
+        if ihost.peers:
+            fields.append('peers')
 
     # Do not display the trailing '+' which indicates the audit iterations
     if ihost.install_state_info:
@@ -51,29 +55,49 @@ def _print_ihost_show(ihost):
         ihost.install_state = ihost.install_state.rstrip('+')
 
     data_list = [(f, getattr(ihost, f, '')) for f in fields]
-    data_list += [(f, getattr(ihost, f, '')) for f in optional_fields
+    if optional_fields:
-                  if hasattr(ihost, f)]
+        data_list += [(f, getattr(ihost, f, '')) for f in optional_fields
+                      if hasattr(ihost, f)]
+
     data = dict(data_list)
-    ordereddata = OrderedDict(sorted(data.items(), key=lambda t: t[0]))
+
-    utils.print_dict(ordereddata, wrap=72)
+    utils.print_dict_with_format(data, wrap=72, output_format=output_format)
 
 
 @utils.arg('hostnameorid', metavar='<hostname or id>',
            help="Name or ID of host")
+@utils.arg('--column',
+           action='append',
+           default=[],
+           help="Specify the column(s) to include, can be repeated")
+@utils.arg('--format',
+           choices=['table', 'yaml', 'value'],
+           help="specify the output format, defaults to table")
 def do_host_show(cc, args):
     """Show host attributes."""
     ihost = ihost_utils._find_ihost(cc, args.hostnameorid)
-    _print_ihost_show(ihost)
+    _print_ihost_show(ihost, args.column, args.format)
 
 
+@utils.arg('--column',
+           action='append',
+           default=[],
+           help="Specify the column(s) to include, can be repeated")
+@utils.arg('--format',
+           choices=['table', 'yaml', 'value'],
+           help="specify the output format, defaults to table")
 def do_host_list(cc, args):
     """List hosts."""
     ihosts = cc.ihost.list()
-    field_labels = ['id', 'hostname', 'personality',
+
-                    'administrative', 'operational', 'availability']
+    if args.column:
-    fields = ['id', 'hostname', 'personality',
+        fields = args.column
-              'administrative', 'operational', 'availability']
+    else:
-    utils.print_list(ihosts, fields, field_labels, sortby=0)
+        fields = ['id', 'hostname', 'personality', 'administrative',
+                  'operational', 'availability']
+
+    utils.print_list(ihosts, fields, fields, sortby=0,
+                     output_format=args.format)
 
 
 def do_host_upgrade_list(cc, args):

sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter_shell.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013-2017 Wind River Systems, Inc.
+# Copyright (c) 2013-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -38,6 +38,13 @@ def do_service_parameter_show(cc, args):
 @utils.arg('--name',
            metavar='<name>',
            help="Search by parameter name")
+@utils.arg('--column',
+           action='append',
+           default=[],
+           help="Specify the column(s) to include, can be repeated")
+@utils.arg('--format',
+           choices=['table', 'yaml', 'value'],
+           help="specify the output format, defaults to table")
 def do_service_parameter_list(cc, args):
     """List Service parameters."""
     query = None
@@ -47,11 +54,14 @@ def do_service_parameter_list(cc, args):
             query = k + '=' + v
     parameters = cc.service_parameter.list(q=options.cli_to_array(query))
 
-    field_labels = ['uuid', 'service', 'section', 'name', 'value',
+    if args.column:
-                    'personality', 'resource']
+        fields = args.column
-    fields = ['uuid', 'service', 'section', 'name', 'value',
+    else:
-              'personality', 'resource']
+        fields = ['uuid', 'service', 'section', 'name', 'value', 'personality',
-    utils.print_list(parameters, fields, field_labels, sortby=None)
+                  'resource']
+
+    utils.print_list(parameters, fields, fields, sortby=None,
+                     output_format=args.format)
 
 
 @utils.arg('uuid',

sysinv/sysinv/centos/build_srpm.data

@@ -1,2 +1,2 @@
 SRC_DIR="sysinv"
-TIS_PATCH_VER=338
+TIS_PATCH_VER=339

sysinv/sysinv/centos/sysinv.spec

@@ -139,6 +139,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_bindir}/sysinv-upgrade
 %{_bindir}/sysinv-puppet
 %{_bindir}/sysinv-helm
+%{_bindir}/sysinv-utils
 
 %package wheels
 Summary: %{name} wheels

sysinv/sysinv/opensuse/sysinv.spec

@@ -161,7 +161,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_bindir}/sysinv-upgrade
 %{_bindir}/sysinv-puppet
 %{_bindir}/sysinv-helm
-
+%{_bindir}/sysinv-utils
 %pre
 %service_add_pre sysinv-api.service sysinv-api.target
 %service_add_pre sysinv-conductor.service sysinv-conductor.target

sysinv/sysinv/sysinv/setup.cfg

@@ -36,6 +36,7 @@ console_scripts =
     sysinv-upgrade = sysinv.cmd.upgrade:main
     sysinv-puppet = sysinv.cmd.puppet:main
     sysinv-helm = sysinv.cmd.helm:main
+    sysinv-utils = sysinv.cmd.utils:main
 
 systemconfig.puppet_plugins = 
     001_platform = sysinv.puppet.platform:PlatformPuppet

sysinv/sysinv/sysinv/sysinv/cmd/utils.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import sys
+import yaml
+
+from sysinv.common import constants
+from sysinv.common import service
+from sysinv.db import api
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+
+# Log and config
+LOG = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+
+
+def create_host_overrides(filename):
+    try:
+        dbapi = api.get_instance()
+
+        # Get the DNS info
+        dns = dbapi.idns_get_one()
+        dns_list = dns.nameservers.split(',')
+        data = {'dns_servers': dns_list}
+
+        # Get the address pool info
+        pools = dbapi.address_pools_get_all()
+        for pool in pools:
+            subnet = pool.network + '/' + str(pool.prefix)
+            range_start = pool.ranges[0][0].encode('utf-8')
+            range_end = pool.ranges[0][-1].encode('utf-8')
+
+            if pool.name == 'pxeboot':
+                pool_data = {'pxeboot_subnet': subnet,
+                             'pxeboot_start_address': range_start,
+                             'pxeboot_end_address': range_end,
+                             'pxeboot_floating_address': pool.floating_address,
+                             'pxeboot_node_0_address': pool.controller0_address,
+                             'pxeboot_node_1_address': pool.controller1_address
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'management':
+                pool_data = {'management_subnet': subnet,
+                             'management_start_address': range_start,
+                             'management_end_address': range_end,
+                             'management_floating_address': pool.floating_address,
+                             'management_node_0_address': pool.controller0_address,
+                             'management_node_1_address': pool.controller1_address
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'multicast-subnet':
+                pool_data = {'management_multicast_subnet': subnet,
+                             'management_multicast_start_address': range_start,
+                             'management_multicast_end_address': range_end
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'cluster-host-subnet':
+                pool_data = {'cluster_host_subnet': subnet,
+                             'cluster_host_start_address': range_start,
+                             'cluster_host_end_address': range_end,
+                             'cluster_host_floating_address': pool.floating_address,
+                             'cluster_host_node_0_address': pool.controller0_address,
+                             'cluster_host_node_1_address': pool.controller1_address
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'cluster-pod-subnet':
+                pool_data = {'cluster_pod_subnet': subnet,
+                             'cluster_pod_start_address': range_start,
+                             'cluster_pod_end_address': range_end
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'cluster-service-subnet':
+                pool_data = {'cluster_service_subnet': subnet,
+                             'cluster_sevice_start_address': range_start,
+                             'cluster_service_end_address': range_end
+                             }
+                data.update(pool_data)
+
+            elif pool.name == 'oam':
+                pool_data = {'external_oam_subnet': subnet,
+                             'external_oam_start_address': range_start,
+                             'external_oam_end_address': range_end,
+                             'external_oam_floating_address': pool.floating_address,
+                             'external_oam_gateway_address': pool.gateway_address
+                             }
+                data.update(pool_data)
+
+                system = dbapi.isystem_get_one()
+                if system.system_mode != constants.SYSTEM_MODE_SIMPLEX:
+                    pool_data = {'external_oam_node_0_address': pool.controller0_address,
+                                 'external_oam_node_1_address': pool.controller1_address,
+                                 }
+                    data.update(pool_data)
+
+        # Get the docker no-proxy info if it exists
+        no_proxy = dbapi.service_parameter_get_one(service='docker',
+                                                   name='no_proxy')
+        if no_proxy:
+            # Remove the open and close parenthesis if address is IPV6
+            _value = no_proxy.value.strip("[]")
+            no_proxy_list = _value.split(',')
+            data.update({'docker_no_proxy': no_proxy_list})
+
+        # Get the docker http_proxy info if it exists
+        http_proxy = dbapi.service_parameter_get_one(service='docker',
+                                                     name='http_proxy')
+        if http_proxy:
+            data.update({'docker_http_proxy': http_proxy.value})
+
+        # Get the docker https_proxy info if it exists
+        https_proxy = dbapi.service_parameter_get_one(service='docker',
+                                                      name='https_proxy')
+        if https_proxy:
+            data.update({'docker_https_proxy': https_proxy.value})
+
+        # Save collected information in file
+        with open(filename, 'w') as outfile:
+            yaml.safe_dump(data, outfile, default_flow_style=False)
+    except Exception as e:
+        LOG.error(e)
+
+
+def add_action_parsers(subparsers):
+
+    parser = subparsers.add_parser('create-host-overrides')
+    parser.set_defaults(func=create_host_overrides)
+    parser.add_argument('filename', nargs='?')
+
+
+CONF.register_cli_opt(
+    cfg.SubCommandOpt('action',
+                      title='actions',
+                      help='Perform sysinv operations',
+                      handler=add_action_parsers))
+
+
+def main():
+    service.prepare_service(sys.argv)
+
+    if CONF.action.name == 'create-host-overrides':
+        if not CONF.action.filename:
+            LOG.error("filename is required")
+        else:
+            CONF.action.func(CONF.action.filename)
+    else:
+        CONF.action.func()

sysinv/sysinv/sysinv/sysinv/conductor/openstack.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013-2018 Wind River Systems, Inc.
+# Copyright (c) 2013-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -452,6 +452,19 @@ class OpenStackOperator(object):
                       "have payload", secret_ref)
             return None
 
+    def get_barbican_secrets_by_name(self, name, use_openstack=False):
+        if use_openstack:
+            client = self._get_barbicanclient(
+                service_config=OPENSTACK_CONFIG)
+        else:
+            client = self._get_barbicanclient()
+        try:
+            secret_list = client.secrets.list(name=name)
+            return secret_list
+        except Exception:
+            LOG.error("Unable to find Barbican secrets %s", name)
+            return None
+
     def get_barbican_secret_by_name(self, context, name):
         try:
             client = self._get_barbicanclient()