From c4eab35f359a5afc68c108ad2c82377029f720bc Mon Sep 17 00:00:00 2001 From: Helena McGough Date: Mon, 27 May 2019 15:57:56 +0000 Subject: [PATCH] Updating flavor creation and management to support usage of generic RSD flavors - Implemented new functionality - Updated existing unit tests Change-Id: Ib7449bec27db7f2636485b0ee7658fd2a5e471e5 Signed-off-by: Helena McGough --- .../tests/virt/rsd/test_driver.py | 103 ++++------ rsd_virt_for_nova/virt/rsd/driver.py | 194 +++++++++--------- .../virt/rsd/flavor_management.py | 2 + 3 files changed, 141 insertions(+), 158 deletions(-) diff --git a/rsd_virt_for_nova/tests/virt/rsd/test_driver.py b/rsd_virt_for_nova/tests/virt/rsd/test_driver.py index d82193d..2fb2408 100644 --- a/rsd_virt_for_nova/tests/virt/rsd/test_driver.py +++ b/rsd_virt_for_nova/tests/virt/rsd/test_driver.py @@ -340,16 +340,17 @@ class TestRSDDriver(base.BaseTestCase): mock_context = context.get_admin_context() self.RSD.rsd_flavors = { self.flavor.flavorid: { - 'rsd_systems': { - '/redfish/v1/Chassis/Chassis1': - self.system_inst.path + 'rsd_systems': self.system_inst.path } } - } + self.RSD.system_rp_id = {'mock_rp_id': '/redfish/v1/Systems/System1'} image_meta = objects.ImageMeta.from_dict(self.test_image_meta) # Run spawning test self.RSD.spawn(mock_context, self.inst1, image_meta, - [], None, {}) + [], None, {'mock_rp_id': { + 'generation': 1, + 'resources': {'VCPUS': 1, + 'MEM_MB': 4126}}}) # Confirm that a node instances is spawned and the physical composed # node is powered on @@ -359,7 +360,7 @@ class TestRSDDriver(base.BaseTestCase): self.assertEquals(node.composed_node_state.lower(), 'assembled') self.assertEquals(node.power_state.lower(), 'off') self.assertEquals(self.inst1.display_description, - json.dumps({"node_identity": "/redfish/v1/Nodes/Node1", + json.dumps({"node_identity": node.name, "node_uuid": node.uuid})) power_on.assert_called_once_with(mock_context, self.inst1, None) @@ -510,8 +511,9 @@ class TestRSDDriver(base.BaseTestCase): self.RSD.driver.PODM.get_chassis_collection.assert_called() chas_col.get_member.assert_called_with('/redfish/v1/Chassis/Chassis1') check_chas.assert_called_with(self.chassis_inst) - create_child_inv.assert_called_once_with('/redfish/v1/Systems/System1') - create_inv.assert_called_once_with(check_chas.return_value) + create_child_inv.assert_called_once_with( + ['/redfish/v1/Systems/System1']) + create_inv.assert_called_once_with(1) @mock.patch.object(driver.RSDDriver, 'create_child_inventory') @mock.patch.object(driver.RSDDriver, 'create_inventory') @@ -692,7 +694,7 @@ class TestRSDDriver(base.BaseTestCase): @mock.patch.object(driver.RSDDriver, 'conv_GiB_to_MiB') @mock.patch.object(driver.RSDDriver, 'get_sys_memory_info') @mock.patch.object(driver.RSDDriver, 'get_sys_proc_info') - def test_create_inventory(self, sys_proc_info, sys_mem_info, + def test_create_child_inventory(self, sys_proc_info, sys_mem_info, conv_mem): """Test creating a inventory for a provider tree.""" # Setup test to successfully create inventory @@ -700,7 +702,7 @@ class TestRSDDriver(base.BaseTestCase): self.system_inst.memory_summary.total_system_memory_gib sys_proc_info.return_value = \ self.system_inst.json['ProcessorSummary']['Count'] - inv = self.RSD.create_inventory([self.system_inst.identity]) + inv = self.RSD.create_child_inventory([self.system_inst.identity]) # Check that the correct functions are called and the inventory # is generated correctly @@ -723,27 +725,15 @@ class TestRSDDriver(base.BaseTestCase): 'allocation_ratio': 1} }) - @mock.patch.object(driver.RSDDriver, 'conv_GiB_to_MiB') @mock.patch.object(fields.ResourceClass, 'normalize_name') - def test_create_child_inventory(self, norm_name, conv_mem): + def test_create_inventory(self, norm_name): """Test creating inventory for the child RP's.""" # Set up a test to create the inventory for child resource providers - sys_col = self.RSD.driver.PODM.get_system_collection.return_value - sys_col.get_member.return_value = self.system_inst - mem = conv_mem.return_value - 512 - proc = self.system_inst.json['ProcessorSummary']['Count'] - flav_id = str(mem) + 'MB-' + str(proc) + 'vcpus' - child_inv = self.RSD.create_child_inventory( - '/redfish/v1/Systems/System1') + child_inv = self.RSD.create_inventory(1) # Check that the correct functions are called and the inventory # is generated correctly - self.RSD.driver.PODM.get_system_collection.assert_called_once() - sys_col.get_member.assert_called_once_with( - '/redfish/v1/Systems/System1') - conv_mem.assert_called_once_with( - self.system_inst.memory_summary.total_system_memory_gib) - norm_name.assert_called_once_with(flav_id) + norm_name.assert_called_once_with('RSD') self.assertEqual(child_inv, {norm_name.return_value: { 'total': 1, 'reserved': 0, @@ -776,66 +766,63 @@ class TestRSDDriver(base.BaseTestCase): @mock.patch.object(flavor_management.FlavorManager, '_create_request_url') @mock.patch.object(driver, 'requests') @mock.patch.object(fields.ResourceClass, 'normalize_name') - @mock.patch.object(driver.RSDDriver, 'check_chassis_systems') - def test_create_flavors(self, check_chas, norm_name, reqs, create_url): + def test_create_flavors(self, norm_name, reqs, create_url): """Test successfully creating a new flavor.""" sys_col = self.RSD.driver.PODM.get_system_collection.return_value sys_col.get_member.return_value = self.system_inst - chas_col = self.RSD.driver.PODM.get_chassis_collection.return_value - chas_col.members_identities = ['/redfish/v1/Chassis/Chassis1'] - chas_col.get_member.return_value = self.chassis_inst - check_chas.return_value = ['/redfish/v1/Systems/System1'] + sys_col.members_identities = ['/redfish/v1/Systems/System1'] self.RSD._url_base = "mock_url_base" self.RSD.headers = "headers" - mem = self.RSD.conv_GiB_to_MiB( - self.system_inst.memory_summary.total_system_memory_gib) - 512 - proc = self.system_inst.json['ProcessorSummary']['Count'] - flav_id = str(mem) + 'MB-' + str(proc) + 'vcpus' spec = str('resources:' + norm_name.return_value) mock_specs = { 'extra_specs': { spec: '1'}} + response = {"flavors": [ + {"id": "1", + "links": [ + {"href": "http://openstack.example.com/v2/6f70656e/flavors/1", + "rel": "self" + }, + {"href": "http://openstack.example.com/6f70656e/flavors/1", + "rel": "bookmark" + }], + "name": "m1.tiny" + }, + {"id": "2", + "links": [ + {"href": "http://openstack.example.com/v2/6f70656e/flavors/2", + "rel": "self" + }, + {"href": "http://openstack.example.com/6f70656e/flavors/2", + "rel": "bookmark" + }], + "name": "m1.small" + }] + } + resp = FakeResponse(response, 201) + reqs.get.return_value.text = json.dumps(resp.text) # Run Test self.RSD._create_flavors() self.RSD.driver.PODM.get_system_collection.assert_called_once() - self.RSD.driver.PODM.get_chassis_collection.assert_called_once() - chas_col.get_member.assert_called_with(self.chassis_inst.path) - check_chas.assert_called_with(self.chassis_inst) sys_col.get_member.assert_called_with(self.system_inst.path) - norm_name.assert_called_once_with(flav_id) - create_url.assert_called_once_with(flav_id, 'update') + norm_name.assert_called_once_with('RSD') + create_url.assert_called() reqs.post.assert_called_with(create_url.return_value, data=json.dumps(mock_specs), headers="headers") - self.assertEquals(self.RSD.chas_systems, - {str(self.chassis_inst.path): - [str(self.system_inst.path)]}) - self.assertEquals(self.RSD.rsd_flavors, - {flav_id: { - 'rsd_systems': self.RSD.chas_systems}}) @mock.patch.object(requests, 'post') @mock.patch.object(fields.ResourceClass, 'normalize_name') - @mock.patch.object(driver.RSDDriver, 'check_chassis_systems') - def test_create_flavors_exists(self, check_chas, norm_name, post_req): + def test_create_flavors_failure(self, norm_name, post_req): """Test existing failure to create a new flavor.""" sys_col = self.RSD.driver.PODM.get_system_collection.return_value - sys_col.get_member.return_value = self.system_inst - chas_col = self.RSD.driver.PODM.get_chassis_collection.return_value - chas_col.members_identities = ['/redfish/v1/Chassis/Chassis1'] - chas_col.get_member.return_value = self.chassis_inst - check_chas.return_value = ['/redfish/v1/Systems/System1'] - # Run Test self.RSD._create_flavors() self.RSD.driver.PODM.get_system_collection.assert_called_once() - self.RSD.driver.PODM.get_chassis_collection.assert_called_once() - chas_col.get_member.assert_called_with(self.chassis_inst.path) - check_chas.assert_called_with(self.chassis_inst) - sys_col.get_member.assert_called_with(self.system_inst.path) + sys_col.get_member.assert_not_called() post_req.assert_not_called() @mock.patch.object(flavor_management.FlavorManager, '_create_request_url') diff --git a/rsd_virt_for_nova/virt/rsd/driver.py b/rsd_virt_for_nova/virt/rsd/driver.py index a49b636..4114af2 100644 --- a/rsd_virt_for_nova/virt/rsd/driver.py +++ b/rsd_virt_for_nova/virt/rsd/driver.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -71,7 +70,7 @@ class RSDDriver(driver.ComputeDriver): self.driver = rsd.PODM_connection() self.instances = OrderedDict() self.rsd_flavors = OrderedDict() - self.chas_systems = OrderedDict() + self.system_rp_id = OrderedDict() self._nodes = [] self._composed_nodes = OrderedDict() self.instance_node = None @@ -127,30 +126,39 @@ class RSDDriver(driver.ComputeDriver): # Assembles the composed node and tracks node from the instance COMPOSED_NODE_COL = self.driver.PODM.get_node_collection() node_inst = None + rp_id = '' + nodes = {} flav_id = instance.flavor.flavorid - sys_list = self.rsd_flavors[flav_id]['rsd_systems'][instance.node] - for n in COMPOSED_NODE_COL.members_identities: - try: - node_inst = self.driver.PODM.get_node(n) - except Exception as ex: - LOG.warn("Malformed composed node instance:%s", ex) + sys_list = self.rsd_flavors[flav_id]['rsd_systems'] + for r in allocations.keys(): + values = allocations[r] + for v in values.values(): + if isinstance(v, int) is False: + if 'CUSTOM_RSD' not in v: + rp_id = r + sys_id = self.system_rp_id[rp_id] + if sys_id in sys_list: + for n in COMPOSED_NODE_COL.members_identities: + try: + node_inst = self.driver.PODM.get_node(n) + nodes[node_inst.links.computer_system] = node_inst + except Exception as ex: + LOG.warn("Malformed composed node instance:%s", ex) - if node_inst is not None: - node_state = node_inst.composed_node_state.lower() - p_state = node_inst.power_state.lower() + if sys_id in nodes.keys(): + # Provide nova instance with composed node info + node = nodes[sys_id] + node_state = node.composed_node_state.lower() if node_state == 'assembled': - # Provide nova instance with composed node info - node_sys_id = node_inst.links.computer_system - - if node_sys_id in sys_list: - self.instances[uuid] = instance - self._composed_nodes[uuid] = node_inst - instance.display_description = \ - json.dumps({"node_identity": n, - "node_uuid": node_inst.uuid}) - if p_state == 'off': - self.power_on(context, instance, network_info) - return + self.instances[uuid] = instance + self._composed_nodes[uuid] = node + instance.display_description = \ + json.dumps({"node_identity": node.name, + "node_uuid": node.uuid}) + if node.power_state.lower() == 'off': + self.power_on(context, instance, network_info) + LOG.info("Booted composed node:%s", n) + return raise Exception("Failed to assign composed node for instance.") def destroy(self, context, instance, network_info, block_device_info=None, @@ -257,10 +265,12 @@ class RSDDriver(driver.ComputeDriver): # Removing all RPS that don't have an associated system if s_tree.name not in systems: provider_tree.remove(str(s_tree.uuid)) + elif s_tree.uuid not in self.system_rp_id.keys(): + self.system_rp_id[s_tree.uuid] = s_tree.name - chassis = CHASSIS_COL.get_member(chas_tree.name) - if self.check_chassis_systems(chassis) == []: - provider_tree.remove(str(chas_tree.uuid)) + chassis = CHASSIS_COL.get_member(chas_tree.name) + if self.check_chassis_systems(chassis) == []: + provider_tree.remove(str(chas_tree.uuid)) self._nodes = self._init_nodes() for c in CHASSIS_COL.members_identities: @@ -272,13 +282,14 @@ class RSDDriver(driver.ComputeDriver): if cha_sys != []: for s in cha_sys: - sys_inv = self.create_child_inventory(s) + sys_inv = self.create_child_inventory([s]) try: - provider_tree.new_child(s, nodename) + prov_id = provider_tree.new_child(s, nodename) + self.system_rp_id[prov_id] = s except Exception as ex: LOG.warn("Failed to create new RP: %s", ex) provider_tree.update_inventory(s, sys_inv) - chas_inv = self.create_inventory(cha_sys) + chas_inv = self.create_inventory(len(cha_sys)) provider_tree.update_inventory(nodename, chas_inv) def get_sys_proc_info(self, systems): @@ -372,7 +383,7 @@ class RSDDriver(driver.ComputeDriver): LOG.info("Graceful restart composed node for reboot: %s", key) node_inst.reset_node('GracefulRestart') - def create_inventory(self, system): + def create_child_inventory(self, system): """Function to create provider tree inventory.""" mem_max = 1 proc_max = 1 @@ -400,18 +411,12 @@ class RSDDriver(driver.ComputeDriver): } } - def create_child_inventory(self, system): + def create_inventory(self, no_systems): """Create custom resources for all of the child RP's.""" - SYSTEM_COL = self.driver.PODM.get_system_collection() - sys = SYSTEM_COL.get_member(system) - mem = self.conv_GiB_to_MiB( - sys.memory_summary.total_system_memory_gib) - 512 - proc = sys.json['ProcessorSummary']['Count'] - flav_id = str(mem) + 'MB-' + str(proc) + 'vcpus' - res = fields.ResourceClass.normalize_name(flav_id) + res = fields.ResourceClass.normalize_name('RSD') return { res: { - 'total': 1, + 'total': no_systems, 'reserved': 0, 'min_unit': 1, 'max_unit': 1, @@ -435,68 +440,57 @@ class RSDDriver(driver.ComputeDriver): def _create_flavors(self): """Auto-generate the flavors for the compute systems available.""" SYSTEM_COL = self.driver.PODM.get_system_collection() - CHASSIS_COL = self.driver.PODM.get_chassis_collection() - for c in CHASSIS_COL.members_identities: - chas = CHASSIS_COL.get_member(c) - cha_sys = self.check_chassis_systems(chas) - for s in cha_sys: - sys = SYSTEM_COL.get_member(s) - mem = self.conv_GiB_to_MiB( - sys.memory_summary.total_system_memory_gib) - 512 - proc = sys.json['ProcessorSummary']['Count'] - flav_id = str(mem) + 'MB-' + str(proc) + 'vcpus' - res = fields.ResourceClass.normalize_name(flav_id) - spec = str('resources:' + res) - payload = { - "flavor": { - 'name': 'RSD-' + flav_id, - 'id': flav_id, - 'ram': mem, - 'vcpus': proc, - 'disk': 0 + for s in SYSTEM_COL.members_identities: + sys = SYSTEM_COL.get_member(s) + mem = self.conv_GiB_to_MiB( + sys.memory_summary.total_system_memory_gib) - 512 + proc = sys.json['ProcessorSummary']['Count'] + flav_id = str(mem) + 'MB-' + str(proc) + 'vcpus' + res = fields.ResourceClass.normalize_name('RSD') + spec = str('resources:' + res) + payload = { + "flavor": { + 'name': 'RSD-' + flav_id, + 'id': flav_id, + 'ram': mem, + 'vcpus': proc, + 'disk': 0 + } + } + try: + requests.post( + self._url_base, data=json.dumps(payload), + headers=self.headers) + except Exception as ex: + LOG.warn("Failed to create the new flavor: %s", ex) + try: + extra_pay = { + 'extra_specs': { + spec: '1'} } - } - if flav_id not in self.rsd_flavors.keys(): - try: - requests.post( - self._url_base, data=json.dumps(payload), - headers=self.headers) - except Exception as ex: - LOG.warn("Failed to create the new flavor: %s", ex) - try: - extra_pay = { - 'extra_specs': { - spec: '1'} - } - update_url = self.flavor_manager._create_request_url( - flav_id, 'update') - requests.post( - update_url, data=json.dumps(extra_pay), - headers=self.headers) - self.chas_systems[str(chas.path)] = [ - str(sys.path)] - self.rsd_flavors[flav_id] = { - 'rsd_systems': self.chas_systems} - except Exception as ex: - LOG.warn("Failed to add extra_specs:%s", ex) - else: - chassis_ = self.rsd_flavors[flav_id]['rsd_systems'] - if str(chas.path) not in chassis_.keys(): - self.chas_systems[str(chas.path)] = [ - str(sys.path)] - self.rsd_flavors[flav_id] = { - 'rsd_systems': self.chas_systems - } - else: - systems = self.rsd_flavors[ - flav_id]['rsd_systems'][str(chas.path)] - if str(sys.path) not in systems: - systems.append(str(sys.path)) - self.chas_systems[str(chas.path)] = systems - self.rsd_flavors[flav_id] = { - 'rsd_systems': self.chas_systems - } + update_url = self.flavor_manager._create_request_url( + flav_id, 'update') + requests.post( + update_url, data=json.dumps(extra_pay), + headers=self.headers) + except Exception as ex: + LOG.warn("Failed to add extra_specs:%s", ex) + + get_url = self.flavor_manager._create_request_url(None, 'get') + req = requests.get(get_url, headers=self.headers) + f_list = json.loads(req.text)['flavors'] + for f in f_list: + if 'RSD' in f['name'] and \ + mem >= f['ram'] and proc >= f['vcpus']: + if f['id'] not in self.rsd_flavors: + self.rsd_flavors[f['id']] = { + 'rsd_systems': []} + systems = self.rsd_flavors[f['id']]['rsd_systems'] + if s not in systems: + systems.append(s) + self.rsd_flavors[f['id']] = { + 'rsd_systems': systems} def check_flavors(self, collection, systems): """Check if flavors should be deleted based on system removal.""" @@ -529,7 +523,7 @@ class RSDDriver(driver.ComputeDriver): for k in self.rsd_flavors.keys(): rsd_id = self.rsd_flavors[k] - sys_list = self.rsd_flavors[k]['rsd_systems'].values() + sys_list = self.rsd_flavors[k]['rsd_systems'] for s in sys_list: if s not in sys_ids: del_url = self.flavor_manager._create_request_url( diff --git a/rsd_virt_for_nova/virt/rsd/flavor_management.py b/rsd_virt_for_nova/virt/rsd/flavor_management.py index 7a92f2e..5c1118e 100644 --- a/rsd_virt_for_nova/virt/rsd/flavor_management.py +++ b/rsd_virt_for_nova/virt/rsd/flavor_management.py @@ -80,6 +80,8 @@ class FlavorManager(object): url = "{}/flavors/%s".format(endpoint) % (flavorid) elif req_type == 'update': url = "{}/flavors/%s/os-extra_specs".format(endpoint) % (flavorid) + elif req_type == 'get': + url = "{}/flavors/detail".format(endpoint) return url def get_headers(self, auth_token):