diff --git a/ceilometer/network/services/lbaas.py b/ceilometer/network/services/lbaas.py index a01370bf..8da5f79a 100644 --- a/ceilometer/network/services/lbaas.py +++ b/ceilometer/network/services/lbaas.py @@ -16,6 +16,7 @@ import abc import collections +from oslo_config import cfg from oslo_log import log from oslo_utils import timeutils import six @@ -32,8 +33,32 @@ LBStatsData = collections.namedtuple( ['active_connections', 'total_connections', 'bytes_in', 'bytes_out'] ) +LOAD_BALANCER_STATUS_V2 = { + 'offline': 0, + 'online': 1, + 'no_monitor': 3, + 'error': 4, + 'degraded': 5 +} -class LBPoolPollster(base.BaseServicesPollster): + +class BaseLBPollster(base.BaseServicesPollster): + """Base Class for Load Balancer pollster""" + + def __init__(self): + super(BaseLBPollster, self).__init__() + self.lb_version = cfg.CONF.service_types.neutron_lbaas_version + + def get_load_balancer_status_id(self, value): + if self.lb_version == 'v1': + resource_status = self.get_status_id(value) + elif self.lb_version == 'v2': + status = value.lower() + resource_status = LOAD_BALANCER_STATUS_V2.get(status, -1) + return resource_status + + +class LBPoolPollster(BaseLBPollster): """Pollster to capture Load Balancer pool status samples.""" FIELDS = ['admin_state_up', @@ -57,7 +82,7 @@ class LBPoolPollster(base.BaseServicesPollster): for pool in resources: LOG.debug("Load Balancer Pool : %s" % pool) - status = self.get_status_id(pool['status']) + status = self.get_load_balancer_status_id(pool['status']) if status == -1: # unknown status, skip this sample LOG.warning(_("Unknown status %(stat)s received on pool " @@ -126,7 +151,7 @@ class LBVipPollster(base.BaseServicesPollster): ) -class LBMemberPollster(base.BaseServicesPollster): +class LBMemberPollster(BaseLBPollster): """Pollster to capture Load Balancer Member status samples.""" FIELDS = ['admin_state_up', @@ -147,7 +172,7 @@ class LBMemberPollster(base.BaseServicesPollster): for member in resources: LOG.debug("Load Balancer Member : %s" % member) - status = self.get_status_id(member['status']) + status = self.get_load_balancer_status_id(member['status']) if status == -1: LOG.warning(_("Unknown status %(stat)s received on member " "%(id)s, skipping sample") diff --git a/ceilometer/neutron_client.py b/ceilometer/neutron_client.py index b0c9df3c..4161bb3f 100644 --- a/ceilometer/neutron_client.py +++ b/ceilometer/neutron_client.py @@ -25,6 +25,10 @@ SERVICE_OPTS = [ cfg.StrOpt('neutron', default='network', help='Neutron service type.'), + cfg.StrOpt('neutron_lbaas_version', + default='v2', + choices=('v1', 'v2'), + help='Neutron load balancer version.') ] cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') @@ -64,6 +68,7 @@ class Client(object): 'service_type': cfg.CONF.service_types.neutron, } self.client = clientv20.Client(**params) + self.lb_version = cfg.CONF.service_types.neutron_lbaas_version @logged def port_get_all(self): @@ -77,18 +82,33 @@ class Client(object): @logged def pool_get_all(self): - resp = self.client.list_pools() - return resp.get('pools') + resources = [] + if self.lb_version == 'v1': + resp = self.client.list_pools() + resources = resp.get('pools') + elif self.lb_version == 'v2': + resources = self.list_pools_v2() + return resources @logged def member_get_all(self): - resp = self.client.list_members() - return resp.get('members') + resources = [] + if self.lb_version == 'v1': + resp = self.client.list_members() + resources = resp.get('members') + elif self.lb_version == 'v2': + resources = self.list_members_v2() + return resources @logged def health_monitor_get_all(self): - resp = self.client.list_health_monitors() - return resp.get('health_monitors') + resources = [] + if self.lb_version == 'v1': + resp = self.client.list_health_monitors() + resources = resp.get('health_monitors') + elif self.lb_version == 'v2': + resources = self.list_health_monitors_v2() + return resources @logged def pool_stats(self, pool): @@ -118,3 +138,250 @@ class Client(object): def fip_get_all(self): fips = self.client.list_floatingips()['floatingips'] return fips + + def list_pools_v2(self): + """This method is used to get the pools list. + + This method uses Load Balancer v2_0 API to achieve + the detailed list of the pools. + + :returns: The list of the pool resources + """ + pool_status = dict() + resp = self.client.list_lbaas_pools() + temp_pools = resp.get('pools') + resources = [] + pool_listener_dict = self._get_pool_and_listener_ids(temp_pools) + for k, v in pool_listener_dict.items(): + loadbalancer_id = self._get_loadbalancer_id_with_listener_id(v) + status = self._get_pool_status(loadbalancer_id, v) + for k, v in status.items(): + pool_status[k] = v + + for pool in temp_pools: + pool_id = pool.get('id') + pool['status'] = pool_status[pool_id] + pool['lb_method'] = pool.get('lb_algorithm') + pool['status_description'] = pool['status'] + # Based on the LBaaSv2 design, the properties 'vip_id' + # and 'subnet_id' should belong to the loadbalancer resource and + # not to the pool resource. However, because we don't want to + # change the metadata of the pool resource this release, + # we set them to empty values manually. + pool['provider'] = '' + pool['vip_id'] = '' + pool['subnet_id'] = '' + resources.append(pool) + + return resources + + def list_members_v2(self): + """Method is used to list the members info. + + This method is used to get the detailed list of the members + with Load Balancer v2_0 API + + :returns: The list of the member resources + """ + resources = [] + pools = self.client.list_lbaas_pools().get('pools') + for pool in pools: + pool_id = pool.get('id') + listener_id = pool.get('listeners')[0].get('id') + lb_id = self._get_loadbalancer_id_with_listener_id(listener_id) + status = self._get_member_status(lb_id, [listener_id, pool_id]) + resp = self.client.list_lbaas_members(pool_id) + temp_members = resp.get('members') + for member in temp_members: + member['status'] = status[member.get('id')] + member['pool_id'] = pool_id + member['status_description'] = member['status'] + resources.append(member) + return resources + + def list_health_monitors_v2(self): + """Method is used to list the health monitors + + This method is used to get the detailed list of the health + monitors with Load Balancer v2_0 + + :returns: The list of the health monitor resources + """ + resp = self.client.list_lbaas_healthmonitors() + resources = resp.get('healthmonitors') + return resources + + def _get_pool_and_listener_ids(self, pools): + """Method is used to get the mapping between pool and listener + + This method is used to get the pool ids and listener ids + from the pool list. + + :param pools: The list of the polls + :returns: The relationship between pool and listener. + It's a dictionary type. The key of this dict is + the id of pool and the value of it is the id of the first + listener which the pool belongs to + """ + pool_listener_dict = dict() + for pool in pools: + key = pool.get("id") + value = pool.get('listeners')[0].get('id') + pool_listener_dict[key] = value + return pool_listener_dict + + def _retrieve_loadbalancer_status_tree(self, loadbalancer_id): + """Method is used to get the status of a LB. + + This method is used to get the status tree of a specific + Load Balancer. + + :param loadbalancer_id: The ID of the specific Load + Balancer. + :returns: The status of the specific Load Balancer. + It consists of the load balancer and all of its + children's provisioning and operating statuses + """ + lb_status_tree = self.client.retrieve_loadbalancer_status( + loadbalancer_id) + return lb_status_tree + + def _get_loadbalancer_id_with_listener_id(self, listener_id): + """This method is used to get the loadbalancer id. + + :param listener_id: The ID of the listener + :returns: The ID of the Loadbalancer + """ + listener = self.client.show_listener(listener_id) + listener_lbs = listener.get('listener').get('loadbalancers') + loadbalancer_id = listener_lbs[0].get('id') + return loadbalancer_id + + def _get_member_status(self, loadbalancer_id, parent_id): + """Method used to get the status of member resource. + + This method is used to get the status of member + resource belonged to the specific Load Balancer. + + :param loadbalancer_id: The ID of the Load Balancer. + :param parent_id: The parent ID list of the member resource. + For the member resource, the parent_id should be [listener_id, + pool_id]. + :returns: The status dictionary of the member + resource. The key is the ID of the member. The value is + the operating statuse of the member resource. + """ + # FIXME(liamji) the following meters are experimental and + # may generate a large load against neutron api. The future + # enhancements can be tracked against: + # https://review.openstack.org/#/c/218560. + # After it has been merged and the neutron client supports + # with the corresponding apis, will change to use the new + # method to get the status of the members. + resp = self._retrieve_loadbalancer_status_tree(loadbalancer_id) + status_tree = resp.get('statuses').get('loadbalancer') + status_dict = dict() + + listeners_status = status_tree.get('listeners') + for listener_status in listeners_status: + listener_id = listener_status.get('id') + if listener_id == parent_id[0]: + pools_status = listener_status.get('pools') + for pool_status in pools_status: + if pool_status.get('id') == parent_id[1]: + members_status = pool_status.get('members') + for member_status in members_status: + key = member_status.get('id') + # If the item has no the property 'id', skip + # it. + if key is None: + continue + # The situation that the property + # 'operating_status' is none is handled in + # the method get_sample() in lbaas.py. + value = member_status.get('operating_status') + status_dict[key] = value + break + break + + return status_dict + + def _get_listener_status(self, loadbalancer_id): + """Method used to get the status of the listener resource. + + This method is used to get the status of the listener + resources belonged to the specific Load Balancer. + + :param loadbalancer_id: The ID of the Load Balancer. + :returns: The status dictionary of the listener + resource. The key is the ID of the listener resource. The + value is the operating statuse of the listener resource. + """ + # FIXME(liamji) the following meters are experimental and + # may generate a large load against neutron api. The future + # enhancements can be tracked against: + # https://review.openstack.org/#/c/218560. + # After it has been merged and the neutron client supports + # with the corresponding apis, will change to use the new + # method to get the status of the listeners. + resp = self._retrieve_loadbalancer_status_tree(loadbalancer_id) + status_tree = resp.get('statuses').get('loadbalancer') + status_dict = dict() + + listeners_status = status_tree.get('listeners') + for listener_status in listeners_status: + key = listener_status.get('id') + # If the item has no the property 'id', skip + # it. + if key is None: + continue + # The situation that the property + # 'operating_status' is none is handled in + # the method get_sample() in lbaas.py. + value = listener_status.get('operating_status') + status_dict[key] = value + + return status_dict + + def _get_pool_status(self, loadbalancer_id, parent_id): + """Method used to get the status of pool resource. + + This method is used to get the status of the pool + resources belonged to the specific Load Balancer. + + :param loadbalancer_id: The ID of the Load Balancer. + :param parent_id: The parent ID of the pool resource. + :returns: The status dictionary of the pool resource. + The key is the ID of the pool resource. The value is + the operating statuse of the pool resource. + """ + # FIXME(liamji) the following meters are experimental and + # may generate a large load against neutron api. The future + # enhancements can be tracked against: + # https://review.openstack.org/#/c/218560. + # After it has been merged and the neutron client supports + # with the corresponding apis, will change to use the new + # method to get the status of the pools. + resp = self._retrieve_loadbalancer_status_tree(loadbalancer_id) + status_tree = resp.get('statuses').get('loadbalancer') + status_dict = dict() + + listeners_status = status_tree.get('listeners') + for listener_status in listeners_status: + listener_id = listener_status.get('id') + if listener_id == parent_id: + pools_status = listener_status.get('pools') + for pool_status in pools_status: + key = pool_status.get('id') + # If the item has no the property 'id', skip + # it. + if key is None: + continue + # The situation that the property + # 'operating_status' is none is handled in + # the method get_sample() in lbaas.py. + value = pool_status.get('operating_status') + status_dict[key] = value + break + + return status_dict diff --git a/ceilometer/tests/unit/network/services/test_lbaas.py b/ceilometer/tests/unit/network/services/test_lbaas.py index 45e63d25..f6226f5d 100644 --- a/ceilometer/tests/unit/network/services/test_lbaas.py +++ b/ceilometer/tests/unit/network/services/test_lbaas.py @@ -14,6 +14,8 @@ # under the License. import mock + +from oslo_config import cfg from oslo_context import context from oslotest import base from oslotest import mockpatch @@ -32,6 +34,9 @@ class _BaseTestLBPollster(base.BaseTestCase): self.addCleanup(mock.patch.stopall) self.context = context.get_admin_context() self.manager = manager.AgentManager() + cfg.CONF.set_override('neutron_lbaas_version', + 'v1', + group='service_types') plugin_base._get_keystone = mock.Mock() catalog = (plugin_base._get_keystone.session.auth.get_access. return_value.service_catalog) diff --git a/ceilometer/tests/unit/test_neutronclient.py b/ceilometer/tests/unit/test_neutronclient.py index 929b1833..4bf61fc3 100644 --- a/ceilometer/tests/unit/test_neutronclient.py +++ b/ceilometer/tests/unit/test_neutronclient.py @@ -13,6 +13,7 @@ # under the License. import mock + from oslotest import base from ceilometer import neutron_client @@ -23,6 +24,7 @@ class TestNeutronClient(base.BaseTestCase): def setUp(self): super(TestNeutronClient, self).setUp() self.nc = neutron_client.Client() + self.nc.lb_version = 'v1' @staticmethod def fake_ports_list(): diff --git a/ceilometer/tests/unit/test_neutronclient_lbaas_v2.py b/ceilometer/tests/unit/test_neutronclient_lbaas_v2.py new file mode 100644 index 00000000..6f105b4d --- /dev/null +++ b/ceilometer/tests/unit/test_neutronclient_lbaas_v2.py @@ -0,0 +1,302 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from neutronclient.v2_0 import client +from oslotest import base + +from ceilometer import neutron_client + + +class TestNeutronClientLBaaSV2(base.BaseTestCase): + + def setUp(self): + super(TestNeutronClientLBaaSV2, self).setUp() + self.nc = neutron_client.Client() + + @staticmethod + def fake_list_lbaas_pools(): + return { + 'pools': [{ + 'lb_algorithm': 'ROUND_ROBIN', + 'protocol': 'HTTP', + 'description': 'simple pool', + 'admin_state_up': True, + 'tenant_id': '1a3e005cf9ce40308c900bcb08e5320c', + 'healthmonitor_id': None, + 'listeners': [{ + 'id': "35cb8516-1173-4035-8dae-0dae3453f37f" + } + ], + 'members': [{ + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbf858'} + ], + 'id': '4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5', + 'name': 'pool1' + }] + } + + @staticmethod + def fake_list_lbaas_members(): + return { + 'members': [{ + 'weight': 1, + 'admin_state_up': True, + 'subnet_id': '013d3059-87a4-45a5-91e9-d721068ae0b2', + 'tenant_id': '1a3e005cf9ce40308c900bcb08e5320c', + 'address': '10.0.0.8', + 'protocol_port': 80, + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbf858' + }] + } + + @staticmethod + def fake_list_lbaas_healthmonitors(): + return { + 'healthmonitors': [{ + 'admin_state_up': True, + 'tenant_id': '6f3584d5754048a18e30685362b88411', + 'delay': 1, + 'expected_codes': '200,201,202', + 'max_retries': 5, + 'http_method': 'GET', + 'timeout': 1, + 'pools': [{ + 'id': '74aa2010-a59f-4d35-a436-60a6da882819' + }], + 'url_path': '/index.html', + 'type': 'HTTP', + 'id': '0a9ac99d-0a09-4b18-8499-a0796850279a' + }] + } + + @staticmethod + def fake_show_listener(): + return { + 'listener': { + 'default_pool_id': None, + 'protocol': 'HTTP', + 'description': '', + 'admin_state_up': True, + 'loadbalancers': [{ + 'id': 'a9729389-6147-41a3-ab22-a24aed8692b2' + }], + 'tenant_id': '3e4d8bec50a845fcb09e03a4375c691d', + 'connection_limit': 100, + 'protocol_port': 80, + 'id': '35cb8516-1173-4035-8dae-0dae3453f37f', + 'name': '' + } + } + + @staticmethod + def fake_retrieve_loadbalancer_status(): + return { + 'statuses': { + 'loadbalancer': { + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'listeners': [{ + 'id': '35cb8516-1173-4035-8dae-0dae3453f37f', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'pools': [{ + 'id': '4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'members': [{ + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbf858', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE' + }], + 'healthmonitor': { + 'id': '785131d2-8f7b-4fee-a7e7-3196e11b4518', + 'provisioning_status': 'ACTIVE' + } + }] + }] + } + } + } + + @staticmethod + def fake_retrieve_loadbalancer_status_complex(): + return { + 'statuses': { + 'loadbalancer': { + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'listeners': [{ + 'id': '35cb8516-1173-4035-8dae-0dae3453f37f', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'pools': [{ + 'id': '4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'members': [{ + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbf858', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE' + }, + { + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbf969', + 'operating_status': 'OFFLINE', + 'provisioning_status': 'ACTIVE' + }], + 'healthmonitor': { + 'id': '785131d2-8f7b-4fee-a7e7-3196e11b4518', + 'provisioning_status': 'ACTIVE' + } + }, + { + 'id': '4c0a0a5f-cf8f-44b7-b912-957daa8ce6f6', + 'operating_status': 'OFFLINE', + 'provisioning_status': 'ACTIVE', + 'members': [{ + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbfa7a', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE' + }], + 'healthmonitor': { + 'id': '785131d2-8f7b-4fee-a7e7-3196e11b4629', + 'provisioning_status': 'ACTIVE' + } + }] + }, + { + 'id': '35cb8516-1173-4035-8dae-0dae3453f48e', + 'operating_status': 'OFFLINE', + 'provisioning_status': 'ACTIVE', + 'pools': [{ + 'id': '4c0a0a5f-cf8f-44b7-b912-957daa8ce7g7', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE', + 'members': [{ + 'id': 'fcf23bde-8cf9-4616-883f-208cebcbfb8b', + 'operating_status': 'ONLINE', + 'provisioning_status': 'ACTIVE' + }], + 'healthmonitor': { + 'id': '785131d2-8f7b-4fee-a7e7-3196e11b473a', + 'provisioning_status': 'ACTIVE' + } + }] + }] + } + } + } + + @mock.patch.object(client.Client, + 'list_lbaas_pools') + @mock.patch.object(client.Client, + 'show_listener') + @mock.patch.object(neutron_client.Client, + '_retrieve_loadbalancer_status_tree') + def test_list_pools_v2(self, mock_status, mock_show, mock_list): + mock_status.return_value = self.fake_retrieve_loadbalancer_status() + mock_show.return_value = self.fake_show_listener() + mock_list.return_value = self.fake_list_lbaas_pools() + pools = self.nc.list_pools_v2() + self.assertEqual(1, len(pools)) + for pool in pools: + self.assertEqual('ONLINE', pool['status']) + self.assertEqual('ROUND_ROBIN', pool['lb_method']) + + @mock.patch.object(client.Client, + 'list_lbaas_pools') + @mock.patch.object(client.Client, + 'list_lbaas_members') + @mock.patch.object(client.Client, + 'show_listener') + @mock.patch.object(neutron_client.Client, + '_retrieve_loadbalancer_status_tree') + def test_list_members_v2(self, mock_status, mock_show, mock_list_members, + mock_list_pools): + mock_status.return_value = self.fake_retrieve_loadbalancer_status() + mock_show.return_value = self.fake_show_listener() + mock_list_pools.return_value = self.fake_list_lbaas_pools() + mock_list_members.return_value = self.fake_list_lbaas_members() + members = self.nc.list_members_v2() + self.assertEqual(1, len(members)) + for member in members: + self.assertEqual('ONLINE', member['status']) + self.assertEqual('4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5', + member['pool_id']) + + @mock.patch.object(client.Client, + 'list_lbaas_healthmonitors') + def test_list_health_monitors_v2(self, mock_list_healthmonitors): + mock_list_healthmonitors.return_value = ( + self.fake_list_lbaas_healthmonitors()) + healthmonitors = self.nc.list_health_monitors_v2() + self.assertEqual(1, len(healthmonitors)) + for healthmonitor in healthmonitors: + self.assertEqual(5, healthmonitor['max_retries']) + + @mock.patch.object(neutron_client.Client, + '_retrieve_loadbalancer_status_tree') + def test_get_member_status(self, mock_status): + mock_status.return_value = ( + self.fake_retrieve_loadbalancer_status_complex()) + loadbalancer_id = '5b1b1b6e-cf8f-44b7-b912-957daa8ce5e5' + listener_id = '35cb8516-1173-4035-8dae-0dae3453f37f' + pool_id = '4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5' + parent_id = [listener_id, pool_id] + result_status = self.nc._get_member_status(loadbalancer_id, + parent_id) + expected_keys = ['fcf23bde-8cf9-4616-883f-208cebcbf858', + 'fcf23bde-8cf9-4616-883f-208cebcbf969'] + excepted_status = { + 'fcf23bde-8cf9-4616-883f-208cebcbf858': 'ONLINE', + 'fcf23bde-8cf9-4616-883f-208cebcbf969': 'OFFLINE'} + + for key in result_status.keys(): + self.assertIn(key, expected_keys) + self.assertEqual(excepted_status[key], result_status[key]) + + @mock.patch.object(neutron_client.Client, + '_retrieve_loadbalancer_status_tree') + def test_get_pool_status(self, mock_status): + mock_status.return_value = ( + self.fake_retrieve_loadbalancer_status_complex()) + loadbalancer_id = '5b1b1b6e-cf8f-44b7-b912-957daa8ce5e5' + parent_id = '35cb8516-1173-4035-8dae-0dae3453f37f' + result_status = self.nc._get_pool_status(loadbalancer_id, + parent_id) + expected_keys = ['4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5', + '4c0a0a5f-cf8f-44b7-b912-957daa8ce6f6'] + excepted_status = { + '4c0a0a5f-cf8f-44b7-b912-957daa8ce5e5': 'ONLINE', + '4c0a0a5f-cf8f-44b7-b912-957daa8ce6f6': 'OFFLINE'} + + for key in result_status.keys(): + self.assertIn(key, expected_keys) + self.assertEqual(excepted_status[key], result_status[key]) + + @mock.patch.object(neutron_client.Client, + '_retrieve_loadbalancer_status_tree') + def test_get_listener_status(self, mock_status): + mock_status.return_value = ( + self.fake_retrieve_loadbalancer_status_complex()) + loadbalancer_id = '5b1b1b6e-cf8f-44b7-b912-957daa8ce5e5' + result_status = self.nc._get_listener_status(loadbalancer_id) + expected_keys = ['35cb8516-1173-4035-8dae-0dae3453f37f', + '35cb8516-1173-4035-8dae-0dae3453f48e'] + excepted_status = { + '35cb8516-1173-4035-8dae-0dae3453f37f': 'ONLINE', + '35cb8516-1173-4035-8dae-0dae3453f48e': 'OFFLINE'} + + for key in result_status.keys(): + self.assertIn(key, expected_keys) + self.assertEqual(excepted_status[key], result_status[key])