Optimize nova and neutron helm plugins

The dbapi calls in nova and neutron plugins scale linearly with
the number of worker nodes which results in poor performance on
a large number of nodes system.

Currently, the dbapi calls get invoked for each of worker node.
This commit reduces it to a certain number of calls.

Tested stx-openstack upload and apply on a lab with 6 worker nodes.
The time of override generation for nova reduced from 11s to 0.6s
and for neutron reduced from 24s to 0.2s.

Also tested on vbox and lab has "vf" type of interface. Verified
the content of generated overrides are same as before.

Change-Id: I2d19d30b01e3348d6eb60b8e2681e3a30ef93ebc
Partial-Bug: 1886563
Signed-off-by: Angie Wang <angie.wang@windriver.com>

python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py

@@ -28,7 +28,23 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
     AUTH_USERS = ['neutron']
     SERVICE_USERS = ['nova']
 
+    def __init__(self, operator):
+        super(NeutronHelm, self).__init__(operator)
+        self.ports_by_ifaceid = {}
+        self.labels_by_hostid = {}
+        self.ifdatanets_by_ifaceid = {}
+        self.interfaces_by_hostid = {}
+        self.addresses_by_hostid = {}
+
     def get_overrides(self, namespace=None):
+        self.ports_by_ifaceid = self._get_interface_ports()
+        self.labels_by_hostid = self._get_host_labels()
+        self.ifdatanets_by_ifaceid = self._get_interface_datanets()
+        self.interfaces_by_hostid = self._get_host_interfaces(
+            sort_key=self._interface_sort_key)
+        self.addresses_by_hostid = self._get_host_addresses()
+        host_overrides = self._get_per_host_overrides()
+
         overrides = {
             common.HELM_NS_OPENSTACK: {
                 'pod': {
@@ -42,19 +58,19 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
                     },
                     'overrides': {
                         'neutron_ovs-agent': {
-                            'hosts': self._get_per_host_overrides()
+                            'hosts': host_overrides
                         },
                         'neutron_dhcp-agent': {
-                            'hosts': self._get_per_host_overrides()
+                            'hosts': host_overrides
                         },
                         'neutron_l3-agent': {
-                            'hosts': self._get_per_host_overrides()
+                            'hosts': host_overrides
                         },
                         'neutron_metadata-agent': {
-                            'hosts': self._get_per_host_overrides()
+                            'hosts': host_overrides
                         },
                         'neutron_sriov-agent': {
-                            'hosts': self._get_per_host_overrides()
+                            'hosts': host_overrides
                         },
                     },
                     'paste': {
@@ -81,14 +97,13 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
         hosts = self.dbapi.ihost_get_list()
 
         for host in hosts:
-            host_labels = self.dbapi.label_get_by_host(host.id)
+            host_labels = self.labels_by_hostid.get(host.id, [])
             if (host.invprovision in [constants.PROVISIONED,
                                       constants.PROVISIONING] or
                     host.ihost_action in [constants.UNLOCK_ACTION,
                                           constants.FORCE_UNLOCK_ACTION]):
                 if (constants.WORKER in utils.get_personalities(host) and
                         utils.has_openstack_compute(host_labels)):
-
                     hostname = str(host.hostname)
                     host_neutron = {
                         'name': hostname,
@@ -104,7 +119,6 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
                         host_neutron['conf'].update({
                             'auto_bridge_add': self._get_host_bridges(host)})
                     host_list.append(host_neutron)
-
         return host_list
 
     def _interface_sort_key(self, iface):
@@ -129,16 +143,15 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
     def _get_host_bridges(self, host):
         bridges = {}
         index = 0
-        for iface in sorted(self.dbapi.iinterface_get_by_ihost(host.id),
-                            key=self._interface_sort_key):
+        for iface in self.interfaces_by_hostid.get(host.id, []):
             if self._is_data_network_type(iface):
                 if any(dn.datanetwork_network_type in
                        [constants.DATANETWORK_TYPE_FLAT,
                         constants.DATANETWORK_TYPE_VLAN] for dn in
-                       self._get_interface_datanets(iface)):
+                       self.ifdatanets_by_ifaceid.get(iface.id, [])):
                     # obtain the assigned bridge for interface
                     brname = 'br-phy%d' % index
-                    port_name = self._get_interface_port_name(iface)
+                    port_name = self._get_interface_port_name(host, iface)
                     bridges[brname] = port_name.encode('utf8', 'strict')
                     index += 1
         return bridges
@@ -148,13 +161,12 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
         tunnel_types = None
         bridge_mappings = ""
         index = 0
-        for iface in sorted(self.dbapi.iinterface_get_by_ihost(host.id),
-                            key=self._interface_sort_key):
+        for iface in self.interfaces_by_hostid.get(host.id, []):
             if self._is_data_network_type(iface):
                 # obtain the assigned bridge for interface
                 brname = 'br-phy%d' % index
                 if brname:
-                    datanets = self._get_interface_datanets(iface)
+                    datanets = self.ifdatanets_by_ifaceid.get(iface.id, [])
                     for datanet in datanets:
                         dn_name = datanet['datanetwork_name'].strip()
                         LOG.debug('_get_dynamic_ovs_agent_config '
@@ -197,12 +209,11 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
 
     def _get_dynamic_sriov_agent_config(self, host):
         physical_device_mappings = ""
-        for iface in sorted(self.dbapi.iinterface_get_by_ihost(host.id),
-                            key=self._interface_sort_key):
+        for iface in self.interfaces_by_hostid.get(host.id, []):
             if self._is_sriov_network_type(iface):
                 # obtain the assigned datanets for interface
-                datanets = self._get_interface_datanets(iface)
-                port_name = self._get_interface_port_name(iface)
+                datanets = self.ifdatanets_by_ifaceid.get(iface.id, [])
+                port_name = self._get_interface_port_name(host, iface)
                 for datanet in datanets:
                     dn_name = datanet['datanetwork_name'].strip()
                     physical_device_mappings += ('%s:%s,' % (dn_name, port_name))
@@ -260,34 +271,37 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
     def _is_sriov_network_type(self, iface):
         return iface.ifclass == constants.INTERFACE_CLASS_PCI_SRIOV
 
-    def _get_interface_datanets(self, iface):
+    def _get_interface_ports(self):
         """
-        Return the data networks of the supplied interface as a list.
+        Builds a dictionary of ports indexed by interface id
         """
+        ports = {}
+        db_ports = self.dbapi.port_get_list()
+        for port in db_ports:
+            ports.setdefault(port.interface_id, []).append(port)
+        return ports
 
-        ifdatanets = self.dbapi.interface_datanetwork_get_by_interface(
-            iface.uuid)
-        return ifdatanets
-
-    def _get_interface_port_name(self, iface):
+    def _get_interface_port_name(self, host, iface):
         """
         Determine the port name of the underlying device.
         """
         if (iface['iftype'] == constants.INTERFACE_TYPE_VF and iface['uses']):
-            lower_iface = self.dbapi.iinterface_get(iface['uses'][0])
+            for i in self.interfaces_by_hostid.get(host.id, []):
+                if i.ifname == iface['uses'][0]:
+                    lower_iface = i
             if lower_iface:
-                return self._get_interface_port_name(lower_iface)
+                return self._get_interface_port_name(host, lower_iface)
         assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET
-        port = self.dbapi.port_get_by_interface(iface.id)
-        if port:
-            return port[0]['name']
+        ifports = self.ports_by_ifaceid.get(iface.id, [])
+        if ifports:
+            return ifports[0]['name']
 
     def _get_interface_primary_address(self, context, host, iface):
         """
         Determine the primary IP address on an interface (if any).  If multiple
         addresses exist then the first address is returned.
         """
-        for address in self.dbapi.addresses_get_by_host(host.id):
+        for address in self.addresses_by_hostid.get(host.id, []):
             if address.ifname == iface.ifname:
                 return address.address
 

python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+import collections
 import copy
 import os
 
@@ -66,10 +67,36 @@ class NovaHelm(openstack.OpenstackBaseHelm):
     NOVNCPROXY_SERVICE_NAME = 'novncproxy'
     NOVNCPROXY_NODE_PORT = '30680'
 
-    def get_overrides(self, namespace=None):
+    def __init__(self, operator):
+        super(NovaHelm, self).__init__(operator)
+        self.labels_by_hostid = {}
+        self.cpus_by_hostid = {}
+        self.interfaces_by_hostid = {}
+        self.cluster_host_network = None
+        self.interface_networks_by_ifaceid = {}
+        self.addresses_by_hostid = {}
+        self.memory_by_hostid = {}
+        self.ethernet_ports_by_hostid = {}
+        self.ifdatanets_by_ifaceid = {}
+        self.datanets_by_netuuid = {}
+        self.rbd_config = {}
 
+    def get_overrides(self, namespace=None):
+        self.labels_by_hostid = self._get_host_labels()
+        self.cpus_by_hostid = self._get_host_cpus()
+        self.interfaces_by_hostid = self._get_host_interfaces()
+        self.cluster_host_network = self.dbapi.network_get_by_type(
+            constants.NETWORK_TYPE_CLUSTER_HOST)
+        self.interface_networks_by_ifaceid = self._get_interface_networks()
+        self.addresses_by_hostid = self._get_host_addresses()
+        self.memory_by_hostid = self._get_host_imemory()
+        self.ethernet_ports_by_hostid = self._get_host_ethernet_ports()
+        self.ifdatanets_by_ifaceid = self._get_interface_datanets()
+        self.datanets_by_netuuid = self._get_datanetworks()
+        self.rbd_config = self._get_storage_ceph_config()
         ssh_privatekey, ssh_publickey = \
             self._get_or_generate_ssh_keys(self.SERVICE_NAME, common.HELM_NS_OPENSTACK)
+
         overrides = {
             common.HELM_NS_OPENSTACK: {
                 'manifests': self._get_compute_ironic_manifests(),
@@ -319,6 +346,81 @@ class NovaHelm(openstack.OpenstackBaseHelm):
             name='alias', values=alias_config)
         return multistring
 
+    def _get_port_interface_id_index(self, host):
+        """
+        Builds a dictionary of ports for a supplied host indexed by interface id.
+
+        A duplicate of _get_port_interface_id_index() from config/.../sysinv/common/interface.py
+        with modification to use cached data instead of querying the DB.
+        """
+        ports = {}
+        for port in self.ethernet_ports_by_hostid.get(host.id, []):
+            ports[port.interface_id] = port
+        return ports
+
+    def _get_interface_name_index(self, host):
+        """
+        Builds a dictionary of interfaces for a supplied host indexed by interface name.
+
+        A duplicate of _get_interface_name_index() from config/.../sysinv/common/interface.py
+        with modification to use cached data instead of querying the DB.
+        """
+        interfaces = {}
+        for iface in self.interfaces_by_hostid.get(host.id, []):
+            interfaces[iface.ifname] = iface
+        return interfaces
+
+    def _get_interface_name_datanets(self, host):
+        """
+        Builds a dictionary of datanets for a supplied host indexed by interface name.
+
+        A duplicate of _get_interface_name_datanets() from config/.../sysinv/common/interface.py
+        with modification to use cached data instead of querying the DB.
+        """
+        datanets = {}
+        for iface in self.interfaces_by_hostid.get(host.id, []):
+
+            datanetworks = []
+            for ifdatanet in self.ifdatanets_by_ifaceid.get(iface.id, []):
+                datanetworks.append(ifdatanet.datanetwork_uuid)
+
+            datanetworks_list = []
+            for datanetwork in datanetworks:
+                dn = self.datanets_by_netuuid.get(datanetwork)
+                if not dn:
+                    raise exception.DataNetworkNotFound(
+                        datanetwork_uuid=datanetwork)
+                datanetwork_dict = \
+                    {'name': dn.name,
+                     'uuid': dn.uuid,
+                     'network_type': dn.network_type,
+                     'mtu': dn.mtu}
+                if dn.network_type == constants.DATANETWORK_TYPE_VXLAN:
+                    datanetwork_dict.update(
+                        {'multicast_group': dn.multicast_group,
+                         'port_num': dn.port_num,
+                         'ttl': dn.ttl,
+                         'mode': dn.mode})
+                datanetworks_list.append(datanetwork_dict)
+            datanets[iface.ifname] = datanetworks_list
+
+        LOG.debug('_get_interface_name_datanets '
+                  'host=%s, datanets=%s', host.hostname, datanets)
+
+        return datanets
+
+    def _get_address_interface_name_index(self, host):
+        """
+        Builds a dictionary of address lists for a supplied host indexed by interface name.
+
+        A duplicate of _get_address_interface_name_index() from config/.../sysinv/common/interface.py
+        with modification to use cached data instead of querying the DB.
+        """
+        addresses = collections.defaultdict(list)
+        for address in self.addresses_by_hostid.get(host.id, []):
+            addresses[address.ifname].append(address)
+        return addresses
+
     def _update_host_pci_whitelist(self, host, pci_config):
         """
         Generate multistring values containing PCI passthrough
@@ -329,14 +431,10 @@ class NovaHelm(openstack.OpenstackBaseHelm):
         """
         # obtain interface information specific to this host
         iface_context = {
-            'ports': interface._get_port_interface_id_index(
-                self.dbapi, host),
-            'interfaces': interface._get_interface_name_index(
-                self.dbapi, host),
-            'interfaces_datanets': interface._get_interface_name_datanets(
-                self.dbapi, host),
-            'addresses': interface._get_address_interface_name_index(
-                self.dbapi, host),
+            'ports': self._get_port_interface_id_index(host),
+            'interfaces': self._get_interface_name_index(host),
+            'interfaces_datanets': self._get_interface_name_datanets(host),
+            'addresses': self._get_address_interface_name_index(host)
         }
 
         # This host's list of PCI passthrough and SR-IOV device dictionaries
@@ -354,50 +452,24 @@ class NovaHelm(openstack.OpenstackBaseHelm):
 
     def _update_host_storage(self, host, default_config, libvirt_config):
         remote_storage = False
-        labels = self.dbapi.label_get_all(host.id)
-        for label in labels:
+        for label in self.labels_by_hostid.get(host.id, []):
             if (label.label_key == common.LABEL_REMOTE_STORAGE and
                     label.label_value == common.LABEL_VALUE_ENABLED):
                 remote_storage = True
                 break
 
-        rbd_pool = constants.CEPH_POOL_EPHEMERAL_NAME
-        rbd_ceph_conf = os.path.join(constants.CEPH_CONF_PATH,
-                                     constants.SB_TYPE_CEPH_CONF_FILENAME)
-
-        # If NOVA is a service on a ceph-external backend, use the ephemeral_pool
-        # and ceph_conf file that are stored in that DB entry.
-        # If NOVA is not on any ceph-external backend, it must be on the internal
-        # ceph backend with default "ephemeral" pool and default "/etc/ceph/ceph.conf"
-        # config file
-        sb_list = self.dbapi.storage_backend_get_list_by_type(
-            backend_type=constants.SB_TYPE_CEPH_EXTERNAL)
-        if sb_list:
-            for sb in sb_list:
-                if constants.SB_SVC_NOVA in sb.services:
-                    ceph_ext_obj = self.dbapi.storage_ceph_external_get(sb.id)
-                    rbd_pool = sb.capabilities.get('ephemeral_pool')
-                    rbd_ceph_conf = \
-                        constants.CEPH_CONF_PATH + os.path.basename(ceph_ext_obj.ceph_conf)
-
         if remote_storage:
             libvirt_config.update({'images_type': 'rbd',
-                                   'images_rbd_pool': rbd_pool,
-                                   'images_rbd_ceph_conf': rbd_ceph_conf})
+                                   'images_rbd_pool': self.rbd_config.get('rbd_pool'),
+                                   'images_rbd_ceph_conf': self.rbd_config.get('rbd_ceph_conf')})
         else:
             libvirt_config.update({'images_type': 'default'})
 
     def _update_host_addresses(self, host, default_config, vnc_config, libvirt_config):
-        interfaces = self.dbapi.iinterface_get_by_ihost(host.id)
-        addresses = self.dbapi.addresses_get_by_host(host.id)
-        cluster_host_network = self.dbapi.network_get_by_type(
-            constants.NETWORK_TYPE_CLUSTER_HOST)
         cluster_host_iface = None
-        for iface in interfaces:
-            interface_network = {'interface_id': iface.id,
-                                 'network_id': cluster_host_network.id}
+        for iface in self.interfaces_by_hostid.get(host.id, []):
             try:
-                self.dbapi.interface_network_query(interface_network)
+                self._get_interface_network_query(iface.id, self.cluster_host_network.id)
                 cluster_host_iface = iface
             except exception.InterfaceNetworkNotFoundByHostInterfaceNetwork:
                 pass
@@ -406,7 +478,7 @@ class NovaHelm(openstack.OpenstackBaseHelm):
             return
         cluster_host_ip = None
         ip_family = None
-        for addr in addresses:
+        for addr in self.addresses_by_hostid.get(host.id, []):
             if addr.interface_id == cluster_host_iface.id:
                 cluster_host_ip = addr.address
                 ip_family = addr.family
@@ -421,16 +493,13 @@ class NovaHelm(openstack.OpenstackBaseHelm):
         vnc_config.update({'vncserver_proxyclient_address': cluster_host_ip})
 
     def _get_ssh_subnet(self):
-        cluster_host_network = self.dbapi.network_get_by_type(
-            constants.NETWORK_TYPE_CLUSTER_HOST)
-        address_pool = self.dbapi.address_pool_get(cluster_host_network.pool_uuid)
+        address_pool = self.dbapi.address_pool_get(self.cluster_host_network.pool_uuid)
         return '%s/%s' % (str(address_pool.network), str(address_pool.prefix))
 
     def _update_reserved_memory(self, host, default_config):
-        host_memory = self.dbapi.imemory_get_by_ihost(host.id)
         reserved_pages = []
         reserved_host_memory = 0
-        for cell in host_memory:
+        for cell in self.memory_by_hostid.get(host.id, []):
             reserved_4K_pages = 'node:%d,size:4,count:%d' % (
                                 cell.numa_node,
                                 cell.platform_reserved_mib * constants.NUM_4K_PER_MiB)
@@ -484,12 +553,9 @@ class NovaHelm(openstack.OpenstackBaseHelm):
         '''
         # obtain interface information specific to this host
         iface_context = {
-            'ports': interface._get_port_interface_id_index(
-                self.dbapi, host),
-            'interfaces': interface._get_interface_name_index(
-                self.dbapi, host),
-            'interfaces_datanets': interface._get_interface_name_datanets(
-                self.dbapi, host),
+            'ports': self._get_port_interface_id_index(host),
+            'interfaces': self._get_interface_name_index(host),
+            'interfaces_datanets': self._get_interface_name_datanets(host),
         }
 
         # find out the numa_nodes of ports which the physnet(datanetwork) is bound with
@@ -526,12 +592,113 @@ class NovaHelm(openstack.OpenstackBaseHelm):
             numa_nodes = ','.join('%s' % n for n in tunneled_net_numa_nodes)
             per_physnet_numa_config.update({'neutron_tunneled': {'numa_nodes': numa_nodes}})
 
+    def _get_host_ethernet_ports(self):
+        """
+        Builds a dictionary of ethernet ports indexed by host id
+        """
+        ethernet_ports = {}
+        db_ethernet_ports = self.dbapi.ethernet_port_get_list()
+        for port in db_ethernet_ports:
+            ethernet_ports.setdefault(port.host_id, []).append(port)
+        return ethernet_ports
+
+    def _get_host_imemory(self):
+        """
+        Builds a dictionary of memory indexed by host id
+        """
+        memory = {}
+        db_memory = self.dbapi.imemory_get_list()
+        for m in db_memory:
+            memory.setdefault(m.forihostid, []).append(m)
+        return memory
+
+    def _get_host_cpus(self):
+        """
+        Builds a dictionary of cpus indexed by host id
+        """
+        cpus = {}
+        db_cpus = self.dbapi.icpu_get_list()
+        for cpu in db_cpus:
+            cpus.setdefault(cpu.forihostid, []).append(cpu)
+        return cpus
+
+    def _get_host_cpu_list(self, host, function=None, threads=False):
+        """
+        Retrieve a list of CPUs for the host, filtered by function and thread
+        siblings (if supplied)
+        """
+        cpus = []
+        for c in self.cpus_by_hostid.get(host.id, []):
+            if c.thread != 0 and not threads:
+                continue
+            if c.allocated_function == function or not function:
+                cpus.append(c)
+        return cpus
+
+    def _get_storage_ceph_config(self):
+        rbd_pool = constants.CEPH_POOL_EPHEMERAL_NAME
+        rbd_ceph_conf = os.path.join(constants.CEPH_CONF_PATH,
+                                     constants.SB_TYPE_CEPH_CONF_FILENAME)
+
+        # If NOVA is a service on a ceph-external backend, use the ephemeral_pool
+        # and ceph_conf file that are stored in that DB entry.
+        # If NOVA is not on any ceph-external backend, it must be on the internal
+        # ceph backend with default "ephemeral" pool and default "/etc/ceph/ceph.conf"
+        # config file
+        sb_list = self.dbapi.storage_backend_get_list_by_type(
+            backend_type=constants.SB_TYPE_CEPH_EXTERNAL)
+        if sb_list:
+            for sb in sb_list:
+                if constants.SB_SVC_NOVA in sb.services:
+                    ceph_ext_obj = self.dbapi.storage_ceph_external_get(sb.id)
+                    rbd_pool = sb.capabilities.get('ephemeral_pool')
+                    rbd_ceph_conf = \
+                        constants.CEPH_CONF_PATH + os.path.basename(ceph_ext_obj.ceph_conf)
+
+        rbd_config = {
+            'rbd_pool': rbd_pool,
+            'rbd_ceph_conf': rbd_ceph_conf
+        }
+        return rbd_config
+
+    def _get_interface_networks(self):
+        """
+        Builds a dictionary of interface networks indexed by interface id
+        """
+        interface_networks = {}
+
+        db_interface_networks = self.dbapi.interface_network_get_all()
+        for iface_net in db_interface_networks:
+            interface_networks.setdefault(iface_net.interface_id, []).append(iface_net)
+        return interface_networks
+
+    def _get_interface_network_query(self, interface_id, network_id):
+        """
+        Return the interface network of the supplied interface id and network id
+        """
+        for iface_net in self.interface_networks_by_ifaceid.get(interface_id, []):
+            if iface_net.interface_id == interface_id and iface_net.network_id == network_id:
+                return iface_net
+
+        raise exception.InterfaceNetworkNotFoundByHostInterfaceNetwork(
+            interface_id=interface_id, network_id=network_id)
+
+    def _get_datanetworks(self):
+        """
+        Builds a dictionary of datanetworks indexed by datanetwork uuid
+        """
+        datanets = {}
+        db_datanets = self.dbapi.datanetworks_get_all()
+        for datanet in db_datanets:
+            datanets.update({datanet.uuid: datanet})
+        return datanets
+
     def _get_per_host_overrides(self):
         host_list = []
         hosts = self.dbapi.ihost_get_list()
 
         for host in hosts:
-            host_labels = self.dbapi.label_get_by_host(host.id)
+            host_labels = self.labels_by_hostid.get(host.id, [])
             if (host.invprovision in [constants.PROVISIONED,
                                       constants.PROVISIONING] or
                     host.ihost_action in [constants.UNLOCK_ACTION,

python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/openstack.py

@@ -512,6 +512,53 @@ class OpenstackBaseHelm(base.BaseHelm):
                 'name': constants.SB_DEFAULT_NAMES[constants.SB_TYPE_CEPH]})
         }
 
+    def _get_interface_datanets(self):
+        """
+        Builds a dictionary of interface datanetworks indexed by interface id
+        """
+        ifdatanets = {}
+        db_ifdatanets = self.dbapi.interface_datanetwork_get_all()
+        for ifdatanet in db_ifdatanets:
+            ifdatanets.setdefault(ifdatanet.interface_id, []).append(ifdatanet)
+        return ifdatanets
+
+    def _get_host_interfaces(self, sort_key=None):
+        """
+        Builds a dictionary of interfaces indexed by host id
+        """
+        interfaces = {}
+        db_interfaces = self.dbapi.iinterface_get_list()
+        if sort_key:
+            db_interfaces = sorted(db_interfaces, key=sort_key)
+
+        for iface in db_interfaces:
+            interfaces.setdefault(iface.forihostid, []).append(iface)
+        return interfaces
+
+    def _get_host_labels(self):
+        """
+        Builds a dictionary of labels indexed by host id
+        """
+        labels = {}
+        db_labels = self.dbapi.label_get_all()
+        for label in db_labels:
+            labels.setdefault(label.host_id, []).append(label)
+        return labels
+
+    def _get_host_addresses(self):
+        """
+        Builds a dictionary of addresses indexed by host id
+        """
+        addresses = {}
+        db_addresses = self.dbapi.addresses_get_all()
+        db_interfaces = self.dbapi.iinterface_get_list()
+        for addr in db_addresses:
+            for iface in db_interfaces:
+                if iface.id == addr.interface_id:
+                    addresses.setdefault(iface.forihostid, []).append(addr)
+                    break
+        return addresses
+
     def execute_manifest_updates(self, operator):
         """
         Update the elements of the armada manifest.