update model to better match kolla's inventory model

- change groups to store hostnames, not host objects
- change service object to store sub-services and groups
- add sub-service object, contains association with group
- update initial model defaults creation
- update json generator output
- update group unittest
- change name of cli command service list to service listgroups
- add new cli command- service list, which lists sub-services of service
- change group listservices to list the 'leaf-node' of services
- change service listgroups to list the 'leaf-node' services

kollacli/ansible/inventory.py

@@ -37,17 +37,19 @@ COMPUTE_GRP_NAME = 'compute'
 CONTROL_GRP_NAME = 'control'
 NETWORK_GRP_NAME = 'network'
 STORAGE_GRP_NAME = 'storage'
+DATABASE_GRP_NAME = 'database'
 
 DEPLOY_GROUPS = [
     COMPUTE_GRP_NAME,
     CONTROL_GRP_NAME,
     NETWORK_GRP_NAME,
     STORAGE_GRP_NAME,
+    DATABASE_GRP_NAME,
     ]
 
-SERVICE_GROUPS = {
+SERVICES = {
-    'cinder-ctl':   ['cinder-api', 'cinder-scheduler'],
+    'cinder':       ['cinder-api', 'cinder-scheduler', 'cinder-backup',
-    'cinder-data':  ['cinder-backup', 'cinder-volume'],
+                     'cinder-volume'],
     'glance':       ['glance-api', 'glance-registry'],
     'haproxy':      [],
     'heat':         ['heat-api', 'heat-api-cfn', 'heat-engine'],
@@ -58,42 +60,43 @@ SERVICE_GROUPS = {
     'murano':       ['murano-api', 'murano-engine'],
     'mysqlcluster': ['mysqlcluster-api', 'mysqlcluster-mgmt',
                      'mysqlcluster-ndb'],
-    'neutron-ctl':  ['neutron-server'],
+    'neutron':      ['neutron-server', 'neutron-agents'],
-    'neutron-data': ['neutron-agents'],
     'nova':         ['nova-api', 'nova-conductor', 'nova-consoleauth',
                      'nova-novncproxy', 'nova-scheduler'],
     'rabbitmq':     [],
-    'swift-ctl':    ['swift-proxy-server'],
+    'swift':        ['swift-proxy-server', 'swift-account-server',
-    'swift-data':   ['swift-account-server',
                      'swift-container-server', 'swift-object-server'],
     }
 
-DEFAULT_HIERARCHY = {
+DEFAULT_GROUPS = {
-    CONTROL_GRP_NAME: [
+    'cinder':                   CONTROL_GRP_NAME,
-        'cinder-ctl',
+    'glance':                   CONTROL_GRP_NAME,
-        'glance',
+    'haproxy':                  CONTROL_GRP_NAME,
-        'haproxy',
+    'heat':                     CONTROL_GRP_NAME,
-        'heat',
+    'horizon':                  CONTROL_GRP_NAME,
-        'horizon',
+    'keystone':                 CONTROL_GRP_NAME,
-        'keystone',
+    'mariadb':                  CONTROL_GRP_NAME,
-        'nova',
+    'memcached':                CONTROL_GRP_NAME,
-        'memcached',
+    'murano':                   CONTROL_GRP_NAME,
-        'murano',
+    'mysqlcluster':             CONTROL_GRP_NAME,
-        'mysqlcluster',
+    'neutron':                  NETWORK_GRP_NAME,
-        'neutron-ctl',
+    'nova':                     CONTROL_GRP_NAME,
-        'rabbitmq',
+    'rabbitmq':                 CONTROL_GRP_NAME,
-        'swift-ctl',
+    'swift':                    CONTROL_GRP_NAME,
-        ],
-    NETWORK_GRP_NAME: [
-        'neutron-data',
-        ],
-    COMPUTE_GRP_NAME: [],
-    STORAGE_GRP_NAME: [
-        'cinder-data',
-        'swift-data',
-        ]
     }
 
+DEFAULT_OVERRIDES = {
+    'cinder-backup':            STORAGE_GRP_NAME,
+    'cinder-volume':            STORAGE_GRP_NAME,
+    'mysqlcluster-ndb':         DATABASE_GRP_NAME,
+    'neutron-server':           CONTROL_GRP_NAME,
+    'swift-account-server':     STORAGE_GRP_NAME,
+    'swift-container-server':   STORAGE_GRP_NAME,
+    'swift-object-server':      STORAGE_GRP_NAME,
+    }
+
+
+# these groups cannot be deleted, they are required by kolla
 PROTECTED_GROUPS = [COMPUTE_GRP_NAME]
 
 
@@ -163,8 +166,7 @@ class HostGroup(object):
 
     def __init__(self, name):
         self.name = name
-        self.services = []
+        self.hostnames = []
-        self._hosts = {}  # kv = hostname:object
         self.vars = {}
         self.version = self.__class__.class_version
 
@@ -172,37 +174,15 @@ class HostGroup(object):
         pass
 
     def add_host(self, host):
-        if host.name not in self._hosts:
+        if host.name not in self.hostnames:
-            self._hosts[host.name] = host
+            self.hostnames.append(host.name)
-        else:
-            host = self._hosts[host.name]
 
     def remove_host(self, host):
-        if host.name in self._hosts:
+        if host.name in self.hostnames:
-            del self._hosts[host.name]
+            self.hostnames.remove(host.name)
-
-    def add_service(self, servicename):
-        service = Service(servicename)
-        if service not in self.services:
-            self.services.append(service)
-        return service
-
-    def remove_service(self, servicename):
-        for service in self.services:
-            if servicename == service.name:
-                self.services.remove(service)
-
-    def get_hosts(self):
-        return self._hosts.values()
 
     def get_hostnames(self):
-        return self._hosts.keys()
+        return self.hostnames
-
-    def get_servicenames(self):
-        names = []
-        for service in self.services:
-            names.append(service.name)
-        return names
 
     def get_vars(self):
         return self.vars.copy()
@@ -231,16 +211,58 @@ class Service(object):
 
     def __init__(self, name):
         self.name = name
-        self._hosts = {}   # kv = name:object
+        self._sub_servicenames = []
-        self.containers = SERVICE_GROUPS[name]
+        self._groupnames = []
-        self.vars = {}
+        self._vars = {}
         self.version = self.__class__.class_version
 
     def upgrade(self):
         pass
 
-    def get_hostnames(self):
+    def add_groupname(self, groupname):
-        return self._hosts.keys()
+        if groupname not in self._groupnames:
+            self._groupnames.append(groupname)
+
+    def remove_groupname(self, groupname):
+        if groupname in self._groupnames:
+            self._groupnames.remove(groupname)
+
+    def get_groupnames(self):
+        return self._groupnames
+
+    def get_sub_servicenames(self):
+        return self._sub_servicenames
+
+    def add_sub_servicename(self, sub_servicename):
+        if sub_servicename not in self._sub_servicenames:
+            self._sub_servicenames.append(sub_servicename)
+
+    def get_vars(self):
+        return self._vars.copy()
+
+
+class SubService(object):
+    class_version = 1
+
+    def __init__(self, name):
+        self.name = name
+        self._groupnames = []
+        self._vars = {}
+        self.version = self.__class__.class_version
+
+    def upgrade(self):
+        pass
+
+    def add_groupname(self, groupname):
+        if groupname not in self._groupnames:
+            self._groupnames.append(groupname)
+
+    def remove_groupname(self, groupname):
+        if groupname in self._groupnames:
+            self._groupnames.remove(groupname)
+
+    def get_groupnames(self):
+        return self._groupnames
 
     def get_vars(self):
         return self.vars.copy()
@@ -250,8 +272,10 @@ class Inventory(object):
     class_version = 1
 
     def __init__(self):
-        self._groups = {}  # kv = name:object
+        self._groups = {}           # kv = name:object
-        self._hosts = {}   # kv = name:object
+        self._hosts = {}            # kv = name:object
+        self._services = {}         # kv = name:object
+        self._sub_services = {}     # kv = name:object
         self.vars = {}
         self.version = self.__class__.class_version
         self.remote_mode = True
@@ -334,13 +358,26 @@ class Inventory(object):
                     pass
 
     def _create_default_inventory(self):
-        for (group_name, service_names) in DEFAULT_HIERARCHY.items():
-            group = self.add_group(group_name)
 
-            # add services
+        # create the default groups
-            for service_name in service_names:
+        for groupname in DEPLOY_GROUPS:
-                group.add_service(service_name)
+            self.add_group(groupname)
-            self._groups[group_name] = group
+
+        # create the default services/sub_services & their default groups
+        for svcname in SERVICES:
+            svc = self.create_service(svcname)
+            default_grpname = DEFAULT_GROUPS[svcname]
+            svc.add_groupname(default_grpname)
+            sub_svcnames = SERVICES[svcname]
+            if sub_svcnames:
+                for sub_svcname in sub_svcnames:
+                    # create a subservice
+                    sub_svc = self.create_sub_service(sub_svcname)
+                    svc.add_sub_servicename(sub_svcname)
+                    if sub_svc.name not in DEFAULT_OVERRIDES:
+                        sub_svc.add_groupname(default_grpname)
+                    else:
+                        sub_svc.add_groupname(DEFAULT_OVERRIDES[sub_svc.name])
 
     def get_hosts(self):
         return self._hosts.values()
@@ -420,7 +457,7 @@ class Inventory(object):
     def add_group(self, groupname):
 
         # Group names cannot overlap with service names:
-        if groupname in SERVICE_GROUPS:
+        if groupname in SERVICES:
             raise CommandError('ERROR: Invalid group name. A service name '
                                'cannot be used for a group name.')
 
@@ -456,12 +493,32 @@ class Inventory(object):
         return groups
 
     def get_group_services(self):
-        """return { groupname : [servicenames] }"""
+        """get groups and their instantiated services
+
+        Instantiated services are those services which can be deployed
+        on a host.
+
+        It can either be:
+        1. a service that has no sub-services
+        2. a sub-service
+
+        return { groupname: [instantiated_servicenames] }
+        """
+
         group_services = {}
+
         for group in self.get_groups():
             group_services[group.name] = []
-            for service in group.services:
+
-                group_services[group.name].append(service.name)
+        for svc in self.get_services():
+            if not svc.get_sub_servicenames():
+                for groupname in svc.get_groupnames():
+                    group_services[groupname].append(svc.name)
+            else:
+                for sub_svcname in svc.get_sub_servicenames():
+                    sub_svc = self.get_sub_service(sub_svcname)
+                    for groupname in sub_svc.get_groupnames():
+                        group_services[groupname].append(sub_svc.name)
         return group_services
 
     def get_group_hosts(self):
@@ -469,46 +526,98 @@ class Inventory(object):
         group_hosts = {}
         for group in self.get_groups():
             group_hosts[group.name] = []
-            for host in group.get_hosts():
+            for hostname in group.get_hostnames():
-                group_hosts[group.name].append(host.name)
+                group_hosts[group.name].append(hostname)
         return group_hosts
 
-    def add_service(self, servicename, groupname):
+    def create_service(self, servicename):
-        if groupname not in self._groups:
+        if servicename not in self._services:
-            raise CommandError('Group name (%s) does not exist'
+            service = Service(servicename)
-                               % groupname)
+            self._services[servicename] = service
+        return self._services[servicename]
 
-        if servicename not in SERVICE_GROUPS.keys():
+    def delete_service(self, servicename):
-            raise CommandError('Service name (%s) does not exist'
+        if servicename in self._services:
-                               % servicename)
+            del self._services[servicename]
 
-        group_services = self.get_group_services()
+    def get_services(self):
-        if servicename not in group_services[groupname]:
+        return self._services.values()
-            group = self._groups[groupname]
-            group.services.append(Service(servicename))
 
-    def remove_service(self, servicename, groupname):
+    def get_service(self, servicename):
-        if groupname not in self._groups:
+        return self._services[servicename]
-            raise CommandError('Group name (%s) does not exist'
-                               % groupname)
 
-        if servicename not in SERVICE_GROUPS.keys():
+    def add_group_to_service(self, groupname, servicename):
-            raise CommandError('Service name (%s) does not exist'
+        service = self.get_service(servicename)
-                               % servicename)
+        if service:
+            service.add_groupname(groupname)
+        else:
+            sub_service = self.get_sub_service(servicename)
+            if sub_service:
+                sub_service.add_groupname(groupname)
+            else:
+                raise CommandError('Service (%s) not found' % servicename)
 
-        group = self._groups[groupname]
+    def remove_group_from_service(self, groupname, servicename):
-        group.remove_service(servicename)
+        service = self.get_service(servicename)
+        if service:
+            service.remove_groupname(groupname)
+        else:
+            sub_service = self.get_sub_service(servicename)
+            if sub_service:
+                sub_service.remove_groupname(groupname)
+            else:
+                raise CommandError('Service (%s) not found' % servicename)
+
+    def create_sub_service(self, sub_servicename):
+        if sub_servicename not in self._sub_services:
+            sub_service = SubService(sub_servicename)
+            self._sub_services[sub_servicename] = sub_service
+        return self._sub_services[sub_servicename]
+
+    def delete_sub_service(self, sub_servicename):
+        if sub_servicename in self._sub_services:
+            del self._sub_services[sub_servicename]
+
+    def get_sub_services(self):
+        return self._sub_services.values()
+
+    def get_sub_service(self, sub_servicename):
+        return self._sub_services[sub_servicename]
+
+    def get_service_sub_services(self):
+        """get services and their sub_services
+
+        return { servicename: [sub_servicenames] }
+        """
+        svc_sub_svcs = {}
+        for service in self.get_services():
+            svc_sub_svcs[service.name] = []
+            svc_sub_svcs[service.name].extend(service.get_sub_servicenames())
+        return svc_sub_svcs
 
     def get_service_groups(self):
-        """return { servicename : groupnames }"""
+        """set instantiated services and their groups
-        service_groups = {}
+
-        group_services = self.get_group_services()
+        Instantiated services are those services which can be deployed
-        for servicename in SERVICE_GROUPS.keys():
+        on a host.
-            service_groups[servicename] = []
+
-            for (groupname, servicenames) in group_services.items():
+        It can either be:
-                if servicename in servicenames:
+        1. a service that has no sub-services
-                    service_groups[servicename].append(groupname)
+        2. a sub-service
-        return service_groups
+
+        return { instantiated_servicename: [groupnames] }
+        """
+        inst_svcs = {}
+        for svc in self.get_services():
+            if not svc.get_sub_servicenames():
+                inst_svcs[svc.name] = []
+                inst_svcs[svc.name].extend(svc.get_groupnames())
+            else:
+                for sub_svcname in svc.get_sub_servicenames():
+                    sub_svc = self.get_sub_service(sub_svcname)
+                    inst_svcs[sub_svcname] = []
+                    inst_svcs[sub_svcname].extend(sub_svc.get_groupnames())
+        return inst_svcs
 
     def set_deploy_mode(self, remote_flag):
         if not remote_flag and len(self._hosts) > 1:
@@ -557,26 +666,17 @@ class Inventory(object):
             jdict[group.name]['children'] = []
             jdict[group.name]['vars'] = group.get_vars()
 
-        # add services
+        # add top-level services and what groups they are in
-        services = []
+        for service in self.get_services():
-        for group in self.get_groups():
+            jdict[service.name] = {}
-            for service in group.services:
+            jdict[service.name]['children'] = service.get_groupnames()
-                if service.name not in jdict:
-                    services.append(service)
-                    jdict[service.name] = {}
-                    jdict[service.name]['vars'] = service.get_vars()
-                    jdict[service.name]['children'] = []
-                if group.name not in jdict[service.name]['children']:
-                    jdict[service.name]['children'].append(group.name)
 
-        # add containers
+        # add sub-services and their groups
-        for service in services:
+        for sub_svc in self.get_sub_services():
-            for containername in service.containers:
+            jdict[sub_svc.name] = {}
-                jdict[containername] = {}
+            jdict[sub_svc.name]['children'] = sub_svc.get_groupnames()
-                jdict[containername]['children'] = [service.name]
-                jdict[containername]['vars'] = {}
 
-        # temporaily create group containing all hosts. this is needed for
+        # temporarily create group containing all hosts. this is needed for
         # ansible commands that are performed on hosts not yet in groups.
         group = self.add_group('__RESERVED__')
         jdict[group.name] = {}

kollacli/group.py

@@ -13,10 +13,10 @@
 #    under the License.
 import logging
 import traceback
-import utils
 
 from kollacli.ansible.inventory import Inventory
 from kollacli.exceptions import CommandError
+from kollacli import utils
 
 from cliff.command import Command
 from cliff.lister import Lister
@@ -169,7 +169,7 @@ class GroupAddservice(Command):
             servicename = utils.convert_to_unicode(servicename)
 
             inventory = Inventory.load()
-            inventory.add_service(servicename, groupname)
+            inventory.add_group_to_service(groupname, servicename)
             Inventory.save(inventory)
         except CommandError as e:
             raise e
@@ -198,7 +198,7 @@ class GroupRemoveservice(Command):
             servicename = utils.convert_to_unicode(servicename)
 
             inventory = Inventory.load()
-            inventory.remove_service(servicename, groupname)
+            inventory.remove_group_from_service(groupname, servicename)
             Inventory.save(inventory)
         except CommandError as e:
             raise e
@@ -219,7 +219,7 @@ class GroupListservices(Lister):
             group_services = inventory.get_group_services()
             if group_services:
                 for (groupname, servicenames) in group_services.items():
-                    data.append((groupname, servicenames))
+                    data.append((groupname, sorted(servicenames)))
             else:
                 data.append(('', ''))
             return (('Group', 'Services'), sorted(data))

kollacli/service.py

@@ -20,8 +20,8 @@ from kollacli.exceptions import CommandError
 from cliff.lister import Lister
 
 
-class ServiceList(Lister):
+class ServiceListGroups(Lister):
-    """Service List"""
+    """List services and their groups"""
 
     log = logging.getLogger(__name__)
 
@@ -41,3 +41,26 @@ class ServiceList(Lister):
             raise e
         except Exception as e:
             raise Exception(traceback.format_exc())
+
+
+class ServiceList(Lister):
+    """List services and their sub-services"""
+
+    log = logging.getLogger(__name__)
+
+    def take_action(self, parsed_args):
+        try:
+            inventory = Inventory.load()
+
+            data = []
+            service_subsvcs = inventory.get_service_sub_services()
+            if service_subsvcs:
+                for (servicename, sub_svcname) in service_subsvcs.items():
+                    data.append((servicename, sub_svcname))
+            else:
+                data.append(('', ''))
+            return (('Service', 'Sub-Services'), sorted(data))
+        except CommandError as e:
+            raise e
+        except Exception as e:
+            raise Exception(traceback.format_exc())

setup.cfg

@@ -53,7 +53,8 @@ kolla.cli =
     property_set = kollacli.property:PropertySet
     property_clear = kollacli.property:PropertyClear
     property_list = kollacli.property:PropertyList
-	service_list = kollacli.service:ServiceList
+    service_list = kollacli.service:ServiceList
+    service_listgroups = kollacli.service:ServiceListGroups
     setdeploy = kollacli.common:Setdeploy
 
 [extract_messages]

tests/group.py

@@ -22,23 +22,28 @@ class TestFunctional(KollaCliTest):
     group1 = {
         'Group': 'control',
         'Services': [
-            'cinder-ctl',
+            'cinder-api', 'cinder-scheduler',
-            'glance',
+            'glance-api', 'glance-registry',
             'haproxy',
-            'heat',
+            'heat-api', 'heat-api-cfn', 'heat-engine',
             'horizon',
             'keystone',
-            'nova',
+            'mariadb',
             'memcached',
-            'murano',
+            'murano-api', 'murano-engine',
-            'mysqlcluster',
+            'mysqlcluster-api', 'mysqlcluster-mgmt',
-            'rabbitmq'],
+            'neutron-server',
+            'nova-api', 'nova-conductor', 'nova-consoleauth',
+            'nova-novncproxy', 'nova-scheduler',
+            'rabbitmq',
+            'swift-proxy-server',
+            ],
         'Hosts': [],
         }
     group2 = {
         'Group': 'network',
         'Services': [
-            'neutron'],
+            'neutron-agents'],
         'Hosts': [],
         }
     group3 = {
@@ -48,10 +53,19 @@ class TestFunctional(KollaCliTest):
         }
     group4 = {
         'Group': 'storage',
-        'Services': ['cinder-data', 'swift'],
+        'Services': [
+            'cinder-backup', 'cinder-volume',
+            'swift-account-server', 'swift-container-server',
+            'swift-object-server'
+            ],
         'Hosts': [],
         }
-    groups = [group1, group2, group3, group4]
+    group5 = {
+        'Group': 'database',
+        'Services': ['mysqlcluster-ndb'],
+        'Hosts': [],
+        }
+    groups = [group1, group2, group3, group4, group5]
 
     def test_group_add_remove(self):
         group_t1 = {
@@ -134,7 +148,7 @@ class TestFunctional(KollaCliTest):
         groupname = group['Group']
         services = group['Services']
 
-        service1 = 'mysqlcluster'
+        service1 = 'horizon'
         service2 = 'rabbitmq'
 
         services.append(service1)
@@ -176,7 +190,8 @@ class TestFunctional(KollaCliTest):
         cli_groups = json.loads(msg)
         self.assertEqual(len(cli_groups), len(groups),
                          '# of groups in cli not equal to expected groups.' +
-                         '\nexpected: %s, \ncli: %s' % (groups, cli_groups))
+                         '\n\nexpected: %s, \n\ncli: %s'
+                         % (groups, cli_groups))
 
         for cli_group in cli_groups:
             cli_hosts = cli_group['Hosts']
@@ -188,12 +203,12 @@ class TestFunctional(KollaCliTest):
                 self.assertEqual(len(cli_hosts), len(group_hosts),
                                  'Group: %s. # of hosts in cli ' % group_name +
                                  'not equal to expected hosts, ' +
-                                 '\nexpected: %s, \ncli: %s'
+                                 '\n\nexpected: %s, \n\ncli: %s'
                                  % (group_hosts, cli_hosts))
                 for group_host in group_hosts:
                     self.assertIn(group_host, cli_hosts,
                                   'Group: %s' % group_name +
-                                  '\nexpected_hosts: %s, \nnot in cli: %s '
+                                  '\n\nexpected_hosts: %s, \n\nnot in cli: %s '
                                   % (group_host, cli_hosts))
 
         # check services in group