openstack/nova
Code Issues Proposed changes
e39bbdc8af
nova/nova/tests/unit/accelerator/test_cyborg.py
Sean Mooney 1356ef5b57 Cyborg evacuate support 
This change extends the conductor manager
to append the cyborg resource request to the
request spec when performing an evacuate.

This change passes the ARQs to spawn during rebuild
and evacuate. On evacuate the existing ARQs will be deleted
and new ARQs will be created and bound, during rebuild the
existing ARQs are reused.

This change extends the rebuild_instance compute rpcapi
function to carry the arq_uuids. This eliminates the
need to lookup the uuids associated with the arqs assinged
to the instance by quering cyborg.

Co-Authored-By: Wenping Song <songwenping@inspur.com>
Co-Authored-By: Brin Zhang <zhangbailin@inspur.com>

Implements: blueprint cyborg-rebuild-and-evacuate
Change-Id: I147bf4d95e6d86ff1f967a8ce37260730f21d236
2020-09-01 08:41:45 +00:00

420 lines
18 KiB
Python
Raw Blame History

# Copyright 2019 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import mock
from keystoneauth1 import exceptions as ks_exc
from requests.models import Response
from oslo_serialization import jsonutils
from nova.accelerator import cyborg
from nova import context
from nova import exception
from nova import objects
from nova.objects import request_spec
from nova import test
from nova.tests.unit import fake_requests
class CyborgTestCase(test.NoDBTestCase):
def setUp(self):
super(CyborgTestCase, self).setUp()
self.context = context.get_admin_context()
self.client = cyborg.get_client(self.context)
def test_get_client(self):
# Set up some ksa conf options
region = 'MyRegion'
endpoint = 'http://example.com:1234'
self.flags(group='cyborg',
region_name=region,
endpoint_override=endpoint)
ctxt = context.get_admin_context()
client = cyborg.get_client(ctxt)
# Dig into the ksa adapter a bit to ensure the conf options got through
# We don't bother with a thorough test of get_ksa_adapter - that's done
# elsewhere - this is just sanity-checking that we spelled things right
# in the conf setup.
self.assertEqual('accelerator', client._client.service_type)
self.assertEqual(region, client._client.region_name)
self.assertEqual(endpoint, client._client.endpoint_override)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_call_cyborg(self, mock_ksa_get):
mock_ksa_get.return_value = 1 # dummy value
resp, err_msg = self.client._call_cyborg(
self.client._client.get, self.client.DEVICE_PROFILE_URL)
self.assertEqual(resp, 1)
self.assertIsNone(err_msg)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_call_cyborg_keystone_error(self, mock_ksa_get):
mock_ksa_get.side_effect = ks_exc.ClientException
resp, err_msg = self.client._call_cyborg(
self.client._client.get, self.client.DEVICE_PROFILE_URL)
self.assertIsNone(resp)
expected_err = 'Could not communicate with Cyborg.'
self.assertIn(expected_err, err_msg)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_call_cyborg_bad_response(self, mock_ksa_get):
mock_ksa_get.return_value = None
resp, err_msg = self.client._call_cyborg(
self.client._client.get, self.client.DEVICE_PROFILE_URL)
self.assertIsNone(resp)
expected_err = 'Invalid response from Cyborg:'
self.assertIn(expected_err, err_msg)
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
@mock.patch.object(Response, 'json')
def test_get_device_profile_list(self, mock_resp_json, mock_call_cyborg):
mock_call_cyborg.return_value = Response(), None
mock_resp_json.return_value = {'device_profiles': 1} # dummy value
ret = self.client._get_device_profile_list(dp_name='mydp')
self.assertEqual(ret, 1)
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_get_device_profile_list_bad_response(self, mock_call_cyborg):
"If Cyborg cannot be reached or returns bad response, raise exception."
mock_call_cyborg.return_value = (None, 'Some error')
self.assertRaises(exception.DeviceProfileError,
self.client._get_device_profile_list,
dp_name='mydp')
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'_get_device_profile_list')
def test_get_device_profile_groups(self, mock_get_dp_list):
mock_get_dp_list.return_value = [{
"groups": [{
"resources:FPGA": "1",
"trait:CUSTOM_FPGA_CARD": "required"
}],
"name": "mydp",
"uuid": "307076c2-5aed-4f72-81e8-1b42f9aa2ec6"
}]
rg = request_spec.RequestGroup(requester_id='device_profile_0')
rg.add_resource(rclass='FPGA', amount='1')
rg.add_trait(trait_name='CUSTOM_FPGA_CARD', trait_type='required')
expected_groups = [rg]
actual_groups = self.client.get_device_profile_groups('mydp')
self.assertEqual(len(expected_groups), len(actual_groups))
self.assertEqual(expected_groups[0].__dict__,
actual_groups[0].__dict__)
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'_get_device_profile_list')
def test_get_device_profile_groups_no_dp(self, mock_get_dp_list):
# If the return value has no device profiles, raise exception
mock_get_dp_list.return_value = None
self.assertRaises(exception.DeviceProfileError,
self.client.get_device_profile_groups,
dp_name='mydp')
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'_get_device_profile_list')
def test_get_device_profile_groups_many_dp(self, mock_get_dp_list):
# If the returned list has more than one dp, raise exception
mock_get_dp_list.return_value = [1, 2]
self.assertRaises(exception.DeviceProfileError,
self.client.get_device_profile_groups,
dp_name='mydp')
def _get_arqs_and_request_groups(self):
arq_common = {
# All ARQs for an instance have same device profile name.
"device_profile_name": "noprog-dp",
"device_rp_uuid": "",
"hostname": "",
"instance_uuid": "",
"state": "Initial",
}
arq_variants = [
{"device_profile_group_id": 0,
"uuid": "edbba496-3cc8-4256-94ca-dfe3413348eb"},
{"device_profile_group_id": 1,
"uuid": "20125bcb-9f55-4e13-8e8c-3fee30e54cca"},
]
arqs = [dict(arq_common, **variant) for variant in arq_variants]
rg_rp_map = {
'device_profile_0': ['c532cf11-02ed-4b03-9dd8-3e9a454131dc'],
'device_profile_1': ['2c332d7b-daaf-4726-a80d-ecf5212da4b8'],
}
return arqs, rg_rp_map
def _get_bound_arqs(self):
arqs, rg_rp_map = self._get_arqs_and_request_groups()
common = {
'host_name': 'myhost',
'instance_uuid': '15d3acf8-df76-400b-bfc9-484a5208daa1',
}
bindings = {
arqs[0]['uuid']: dict(
common, device_rp_uuid=rg_rp_map['device_profile_0'][0]),
arqs[1]['uuid']: dict(
common, device_rp_uuid=rg_rp_map['device_profile_1'][0]),
}
bound_arq_common = {
"attach_handle_info": {
"bus": "01",
"device": "00",
"domain": "0000",
"function": "0" # will vary function ID later
},
"attach_handle_type": "PCI",
"state": "Bound",
# Devic eprofile name is common to all bound ARQs
"device_profile_name": arqs[0]["device_profile_name"],
**common
}
bound_arqs = [
{'uuid': arq['uuid'],
'device_profile_group_id': arq['device_profile_group_id'],
'device_rp_uuid': bindings[arq['uuid']]['device_rp_uuid'],
**bound_arq_common} for arq in arqs]
for index, bound_arq in enumerate(bound_arqs):
bound_arq['attach_handle_info']['function'] = index # fix func ID
return bindings, bound_arqs
@mock.patch('keystoneauth1.adapter.Adapter.post')
def test_create_arqs_failure(self, mock_cyborg_post):
# If Cyborg returns invalid response, raise exception.
mock_cyborg_post.return_value = None
self.assertRaises(exception.AcceleratorRequestOpFailed,
self.client._create_arqs,
dp_name='mydp')
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'_create_arqs')
def test_create_arq_and_match_rps(self, mock_create_arqs):
# Happy path
arqs, rg_rp_map = self._get_arqs_and_request_groups()
dp_name = arqs[0]["device_profile_name"]
mock_create_arqs.return_value = arqs
ret_arqs = self.client.create_arqs_and_match_resource_providers(
dp_name, rg_rp_map)
# Each value in rg_rp_map is a list. We merge them into a single list.
expected_rp_uuids = sorted(list(
itertools.chain.from_iterable(rg_rp_map.values())))
ret_rp_uuids = sorted([arq['device_rp_uuid'] for arq in ret_arqs])
self.assertEqual(expected_rp_uuids, ret_rp_uuids)
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'_create_arqs')
def test_create_arq_and_match_rps_exception(self, mock_create_arqs):
# If Cyborg response does not contain ARQs, raise
arqs, rg_rp_map = self._get_arqs_and_request_groups()
dp_name = arqs[0]["device_profile_name"]
mock_create_arqs.return_value = None
self.assertRaises(
exception.AcceleratorRequestOpFailed,
self.client.create_arqs_and_match_resource_providers,
dp_name, rg_rp_map)
@mock.patch('keystoneauth1.adapter.Adapter.patch')
def test_bind_arqs(self, mock_cyborg_patch):
bindings, bound_arqs = self._get_bound_arqs()
arq_uuid = bound_arqs[0]['uuid']
patch_list = {}
for arq_uuid, binding in bindings.items():
patch = [{"path": "/" + field,
"op": "add",
"value": value
} for field, value in binding.items()]
patch_list[arq_uuid] = patch
self.client.bind_arqs(bindings)
mock_cyborg_patch.assert_called_once_with(
self.client.ARQ_URL, json=mock.ANY)
called_params = mock_cyborg_patch.call_args.kwargs['json']
self.assertEqual(sorted(called_params), sorted(patch_list))
@mock.patch('nova.accelerator.cyborg._CyborgClient.delete_arqs_by_uuid')
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_bind_arqs_exception(self, mock_call_cyborg, mock_del_arqs):
# If Cyborg returns invalid response, raise exception.
bindings, _ = self._get_bound_arqs()
mock_call_cyborg.return_value = None, 'Some error'
self.assertRaises(exception.AcceleratorRequestBindingFailed,
self.client.bind_arqs, bindings=bindings)
mock_del_arqs.assert_not_called()
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arqs_for_instance(self, mock_cyborg_get):
# Happy path, without only_resolved=True
_, bound_arqs = self._get_bound_arqs()
instance_uuid = bound_arqs[0]['instance_uuid']
query = {"instance": instance_uuid}
content = jsonutils.dumps({'arqs': bound_arqs})
resp = fake_requests.FakeResponse(200, content)
mock_cyborg_get.return_value = resp
ret_arqs = self.client.get_arqs_for_instance(instance_uuid)
mock_cyborg_get.assert_called_once_with(
self.client.ARQ_URL, params=query)
bound_arqs.sort(key=lambda x: x['uuid'])
ret_arqs.sort(key=lambda x: x['uuid'])
for ret_arq, bound_arq in zip(ret_arqs, bound_arqs):
self.assertDictEqual(ret_arq, bound_arq)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arqs_for_instance_exception(self, mock_cyborg_get):
# If Cyborg returns an error code, raise exception
_, bound_arqs = self._get_bound_arqs()
instance_uuid = bound_arqs[0]['instance_uuid']
resp = fake_requests.FakeResponse(404, content='')
mock_cyborg_get.return_value = resp
self.assertRaises(
exception.AcceleratorRequestOpFailed,
self.client.get_arqs_for_instance, instance_uuid)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arqs_for_instance_exception_no_resp(self, mock_cyborg_get):
# If Cyborg returns an error code, raise exception
_, bound_arqs = self._get_bound_arqs()
instance_uuid = bound_arqs[0]['instance_uuid']
content = jsonutils.dumps({'noarqs': 'oops'})
resp = fake_requests.FakeResponse(200, content)
mock_cyborg_get.return_value = resp
self.assertRaisesRegex(
exception.AcceleratorRequestOpFailed,
'Cyborg returned no accelerator requests for ',
self.client.get_arqs_for_instance, instance_uuid)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arqs_for_instance_all_resolved(self, mock_cyborg_get):
# If all ARQs are resolved, return full list
_, bound_arqs = self._get_bound_arqs()
instance_uuid = bound_arqs[0]['instance_uuid']
query = {"instance": instance_uuid}
content = jsonutils.dumps({'arqs': bound_arqs})
resp = fake_requests.FakeResponse(200, content)
mock_cyborg_get.return_value = resp
ret_arqs = self.client.get_arqs_for_instance(
instance_uuid, only_resolved=True)
mock_cyborg_get.assert_called_once_with(
self.client.ARQ_URL, params=query)
bound_arqs.sort(key=lambda x: x['uuid'])
ret_arqs.sort(key=lambda x: x['uuid'])
for ret_arq, bound_arq in zip(ret_arqs, bound_arqs):
self.assertDictEqual(ret_arq, bound_arq)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arqs_for_instance_some_resolved(self, mock_cyborg_get):
# If only some ARQs are resolved, return just the resolved ones
unbound_arqs, _ = self._get_arqs_and_request_groups()
_, bound_arqs = self._get_bound_arqs()
# Create a amixture of unbound and bound ARQs
arqs = [unbound_arqs[0], bound_arqs[0]]
instance_uuid = bound_arqs[0]['instance_uuid']
query = {"instance": instance_uuid}
content = jsonutils.dumps({'arqs': arqs})
resp = fake_requests.FakeResponse(200, content)
mock_cyborg_get.return_value = resp
ret_arqs = self.client.get_arqs_for_instance(
instance_uuid, only_resolved=True)
mock_cyborg_get.assert_called_once_with(
self.client.ARQ_URL, params=query)
self.assertEqual(ret_arqs, [bound_arqs[0]])
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_delete_arqs_for_instance(self, mock_call_cyborg):
# Happy path
mock_call_cyborg.return_value = ('Some Value', None)
instance_uuid = 'edbba496-3cc8-4256-94ca-dfe3413348eb'
self.client.delete_arqs_for_instance(instance_uuid)
mock_call_cyborg.assert_called_once_with(mock.ANY,
self.client.ARQ_URL, params={'instance': instance_uuid})
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_delete_arqs_for_instance_exception(self, mock_call_cyborg):
# If Cyborg returns invalid response, raise exception.
err_msg = 'Some error'
mock_call_cyborg.return_value = (None, err_msg)
instance_uuid = 'edbba496-3cc8-4256-94ca-dfe3413348eb'
exc = self.assertRaises(exception.AcceleratorRequestOpFailed,
self.client.delete_arqs_for_instance, instance_uuid)
expected_msg = ('Failed to delete accelerator requests: ' +
err_msg + ' Instance ' + instance_uuid)
self.assertEqual(expected_msg, exc.format_message())
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_delete_arqs_by_uuid(self, mock_call_cyborg):
# Happy path
mock_call_cyborg.return_value = ('Some Value', None)
_, bound_arqs = self._get_bound_arqs()
arq_uuids = [arq['uuid'] for arq in bound_arqs]
arq_uuid_str = ','.join(arq_uuids)
self.client.delete_arqs_by_uuid(arq_uuids)
mock_call_cyborg.assert_called_once_with(mock.ANY,
self.client.ARQ_URL, params={'arqs': arq_uuid_str})
@mock.patch('nova.accelerator.cyborg.LOG.error')
@mock.patch('nova.accelerator.cyborg._CyborgClient._call_cyborg')
def test_delete_arqs_by_uuid_exception(self, mock_call_cyborg, mock_log):
mock_call_cyborg.return_value = (None, 'Some error')
_, bound_arqs = self._get_bound_arqs()
arq_uuids = [arq['uuid'] for arq in bound_arqs]
arq_uuid_str = ','.join(arq_uuids)
self.client.delete_arqs_by_uuid(arq_uuids)
mock_call_cyborg.assert_called_once_with(mock.ANY,
self.client.ARQ_URL, params={'arqs': arq_uuid_str})
mock_log.assert_called_once_with('Failed to delete ARQs %s',
arq_uuid_str)
@mock.patch('keystoneauth1.adapter.Adapter.get')
def test_get_arq_uuids_for_instance(self, mock_cyborg_get):
# Happy path, without only_resolved=True
_, bound_arqs = self._get_bound_arqs()
instance_uuid = bound_arqs[0]['instance_uuid']
flavor = objects.Flavor(extra_specs={'accel:device_profile': 'dp1'})
instance = objects.Instance(flavor=flavor,
uuid=instance_uuid)
query = {"instance": instance_uuid}
content = jsonutils.dumps({'arqs': bound_arqs})
resp = fake_requests.FakeResponse(200, content)
mock_cyborg_get.return_value = resp
ret_arqs = self.client.get_arq_uuids_for_instance(instance)
mock_cyborg_get.assert_called_once_with(
self.client.ARQ_URL, params=query)
bound_arqs = [bound_arq['uuid'] for bound_arq in bound_arqs]
bound_arqs.sort()
ret_arqs.sort()
self.assertEqual(bound_arqs, ret_arqs)
View Git Blame Copy Permalink