OpenStack Compute (Nova)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

9223 lines
452 KiB

  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. """Unit tests for ComputeManager()."""
  13. import contextlib
  14. import datetime
  15. import time
  16. from cinderclient import exceptions as cinder_exception
  17. from cursive import exception as cursive_exception
  18. import ddt
  19. from eventlet import event as eventlet_event
  20. from eventlet import timeout as eventlet_timeout
  21. from keystoneauth1 import exceptions as keystone_exception
  22. import mock
  23. import netaddr
  24. from oslo_log import log as logging
  25. import oslo_messaging as messaging
  26. from oslo_serialization import jsonutils
  27. from oslo_utils import timeutils
  28. from oslo_utils import uuidutils
  29. import six
  30. import testtools
  31. import nova
  32. from nova.compute import build_results
  33. from nova.compute import manager
  34. from nova.compute import power_state
  35. from nova.compute import resource_tracker
  36. from nova.compute import task_states
  37. from nova.compute import utils as compute_utils
  38. from nova.compute import vm_states
  39. from nova.conductor import api as conductor_api
  40. import nova.conf
  41. from nova import context
  42. from nova.db import api as db
  43. from nova import exception
  44. from nova.network import api as network_api
  45. from nova.network import model as network_model
  46. from nova import objects
  47. from nova.objects import block_device as block_device_obj
  48. from nova.objects import fields
  49. from nova.objects import instance as instance_obj
  50. from nova.objects import migrate_data as migrate_data_obj
  51. from nova.objects import network_request as net_req_obj
  52. from nova import test
  53. from nova.tests import fixtures
  54. from nova.tests.unit.api.openstack import fakes
  55. from nova.tests.unit.compute import fake_resource_tracker
  56. from nova.tests.unit import fake_block_device
  57. from nova.tests.unit import fake_flavor
  58. from nova.tests.unit import fake_instance
  59. from nova.tests.unit import fake_network
  60. from nova.tests.unit import fake_network_cache_model
  61. from nova.tests.unit import fake_notifier
  62. from nova.tests.unit.objects import test_instance_fault
  63. from nova.tests.unit.objects import test_instance_info_cache
  64. from nova.tests import uuidsentinel as uuids
  65. from nova import utils
  66. from nova.virt.block_device import DriverVolumeBlockDevice as driver_bdm_volume
  67. from nova.virt import driver as virt_driver
  68. from nova.virt import event as virtevent
  69. from nova.virt import fake as fake_driver
  70. from nova.virt import hardware
  71. from nova.volume import cinder
  72. CONF = nova.conf.CONF
  73. fake_host_list = [mock.sentinel.host1]
  74. @ddt.ddt
  75. class ComputeManagerUnitTestCase(test.NoDBTestCase):
  76. def setUp(self):
  77. super(ComputeManagerUnitTestCase, self).setUp()
  78. self.compute = manager.ComputeManager()
  79. self.context = context.RequestContext(fakes.FAKE_USER_ID,
  80. fakes.FAKE_PROJECT_ID)
  81. self.useFixture(fixtures.SpawnIsSynchronousFixture())
  82. self.useFixture(fixtures.EventReporterStub())
  83. @mock.patch.object(manager.ComputeManager, '_get_power_state')
  84. @mock.patch.object(manager.ComputeManager, '_sync_instance_power_state')
  85. @mock.patch.object(objects.Instance, 'get_by_uuid')
  86. @mock.patch.object(objects.Migration, 'get_by_instance_and_status')
  87. @mock.patch.object(nova.network.neutronv2.api.API,
  88. 'migrate_instance_start')
  89. def _test_handle_lifecycle_event(self, migrate_instance_start,
  90. mock_get_migration, mock_get,
  91. mock_sync, mock_get_power_state,
  92. transition, event_pwr_state,
  93. current_pwr_state):
  94. event = mock.Mock()
  95. mock_get.return_value = fake_instance.fake_instance_obj(self.context,
  96. task_state=task_states.MIGRATING)
  97. event.get_transition.return_value = transition
  98. mock_get_power_state.return_value = current_pwr_state
  99. self.compute.handle_lifecycle_event(event)
  100. expected_attrs = []
  101. if transition in [virtevent.EVENT_LIFECYCLE_POSTCOPY_STARTED,
  102. virtevent.EVENT_LIFECYCLE_MIGRATION_COMPLETED]:
  103. expected_attrs.append('info_cache')
  104. mock_get.assert_called_once_with(
  105. test.MatchType(context.RequestContext),
  106. event.get_instance_uuid.return_value,
  107. expected_attrs=expected_attrs)
  108. if event_pwr_state == current_pwr_state:
  109. mock_sync.assert_called_with(mock.ANY, mock_get.return_value,
  110. event_pwr_state)
  111. else:
  112. self.assertFalse(mock_sync.called)
  113. migrate_finish_statuses = {
  114. virtevent.EVENT_LIFECYCLE_POSTCOPY_STARTED: 'running (post-copy)',
  115. virtevent.EVENT_LIFECYCLE_MIGRATION_COMPLETED: 'running'
  116. }
  117. if transition in migrate_finish_statuses:
  118. mock_get_migration.assert_called_with(
  119. test.MatchType(context.RequestContext),
  120. mock_get.return_value.uuid,
  121. migrate_finish_statuses[transition])
  122. migrate_instance_start.assert_called_once_with(
  123. test.MatchType(context.RequestContext),
  124. mock_get.return_value,
  125. mock_get_migration.return_value)
  126. else:
  127. mock_get_migration.assert_not_called()
  128. migrate_instance_start.assert_not_called()
  129. def test_handle_lifecycle_event(self):
  130. event_map = {virtevent.EVENT_LIFECYCLE_STOPPED: power_state.SHUTDOWN,
  131. virtevent.EVENT_LIFECYCLE_STARTED: power_state.RUNNING,
  132. virtevent.EVENT_LIFECYCLE_PAUSED: power_state.PAUSED,
  133. virtevent.EVENT_LIFECYCLE_RESUMED: power_state.RUNNING,
  134. virtevent.EVENT_LIFECYCLE_SUSPENDED:
  135. power_state.SUSPENDED,
  136. virtevent.EVENT_LIFECYCLE_POSTCOPY_STARTED:
  137. power_state.PAUSED,
  138. virtevent.EVENT_LIFECYCLE_MIGRATION_COMPLETED:
  139. power_state.PAUSED,
  140. }
  141. for transition, pwr_state in event_map.items():
  142. self._test_handle_lifecycle_event(transition=transition,
  143. event_pwr_state=pwr_state,
  144. current_pwr_state=pwr_state)
  145. def test_handle_lifecycle_event_state_mismatch(self):
  146. self._test_handle_lifecycle_event(
  147. transition=virtevent.EVENT_LIFECYCLE_STOPPED,
  148. event_pwr_state=power_state.SHUTDOWN,
  149. current_pwr_state=power_state.RUNNING)
  150. @mock.patch('nova.objects.Instance.get_by_uuid')
  151. @mock.patch('nova.compute.manager.ComputeManager.'
  152. '_sync_instance_power_state')
  153. @mock.patch('nova.objects.Migration.get_by_instance_and_status',
  154. side_effect=exception.MigrationNotFoundByStatus(
  155. instance_id=uuids.instance, status='running (post-copy)'))
  156. def test_handle_lifecycle_event_postcopy_migration_not_found(
  157. self, mock_get_migration, mock_sync, mock_get_instance):
  158. """Tests a EVENT_LIFECYCLE_POSTCOPY_STARTED scenario where the
  159. migration record is not found by the expected status.
  160. """
  161. inst = fake_instance.fake_instance_obj(
  162. self.context, uuid=uuids.instance,
  163. task_state=task_states.MIGRATING)
  164. mock_get_instance.return_value = inst
  165. event = virtevent.LifecycleEvent(
  166. uuids.instance, virtevent.EVENT_LIFECYCLE_POSTCOPY_STARTED)
  167. with mock.patch.object(self.compute, '_get_power_state',
  168. return_value=power_state.PAUSED):
  169. with mock.patch.object(self.compute.network_api,
  170. 'migrate_instance_start') as mig_start:
  171. self.compute.handle_lifecycle_event(event)
  172. # Since we failed to find the migration record, we shouldn't call
  173. # migrate_instance_start.
  174. mig_start.assert_not_called()
  175. mock_get_migration.assert_called_once_with(
  176. test.MatchType(context.RequestContext), uuids.instance,
  177. 'running (post-copy)')
  178. @mock.patch('nova.compute.utils.notify_about_instance_action')
  179. def test_delete_instance_info_cache_delete_ordering(self, mock_notify):
  180. call_tracker = mock.Mock()
  181. call_tracker.clear_events_for_instance.return_value = None
  182. mgr_class = self.compute.__class__
  183. orig_delete = mgr_class._delete_instance
  184. specd_compute = mock.create_autospec(mgr_class)
  185. # spec out everything except for the method we really want
  186. # to test, then use call_tracker to verify call sequence
  187. specd_compute._delete_instance = orig_delete
  188. specd_compute.host = 'compute'
  189. mock_inst = mock.Mock()
  190. mock_inst.uuid = uuids.instance
  191. mock_inst.save = mock.Mock()
  192. mock_inst.destroy = mock.Mock()
  193. mock_inst.system_metadata = mock.Mock()
  194. def _mark_notify(*args, **kwargs):
  195. call_tracker._notify_about_instance_usage(*args, **kwargs)
  196. def _mark_shutdown(*args, **kwargs):
  197. call_tracker._shutdown_instance(*args, **kwargs)
  198. specd_compute.instance_events = call_tracker
  199. specd_compute._notify_about_instance_usage = _mark_notify
  200. specd_compute._shutdown_instance = _mark_shutdown
  201. mock_bdms = mock.Mock()
  202. specd_compute._delete_instance(specd_compute,
  203. self.context,
  204. mock_inst,
  205. mock_bdms)
  206. methods_called = [n for n, a, k in call_tracker.mock_calls]
  207. self.assertEqual(['clear_events_for_instance',
  208. '_notify_about_instance_usage',
  209. '_shutdown_instance'],
  210. methods_called)
  211. mock_notify.assert_called_once_with(self.context,
  212. mock_inst,
  213. specd_compute.host,
  214. action='delete',
  215. phase='start',
  216. bdms=mock_bdms)
  217. def _make_compute_node(self, hyp_hostname, cn_id):
  218. cn = mock.Mock(spec_set=['hypervisor_hostname', 'id',
  219. 'destroy'])
  220. cn.id = cn_id
  221. cn.hypervisor_hostname = hyp_hostname
  222. return cn
  223. @mock.patch.object(manager.ComputeManager, '_get_resource_tracker')
  224. def test_update_available_resource_for_node(self, get_rt):
  225. rt = mock.Mock(spec_set=['update_available_resource'])
  226. get_rt.return_value = rt
  227. self.compute._update_available_resource_for_node(
  228. self.context,
  229. mock.sentinel.node,
  230. )
  231. rt.update_available_resource.assert_called_once_with(
  232. self.context,
  233. mock.sentinel.node,
  234. )
  235. @mock.patch('nova.compute.manager.LOG')
  236. @mock.patch.object(manager.ComputeManager, '_get_resource_tracker')
  237. def test_update_available_resource_for_node_fail_no_host(self, get_rt,
  238. log_mock):
  239. rt = mock.Mock(spec_set=['update_available_resource'])
  240. exc = exception.ComputeHostNotFound(host=mock.sentinel.host)
  241. rt.update_available_resource.side_effect = exc
  242. get_rt.return_value = rt
  243. # Fake out the RT on the compute manager object so we can assert it's
  244. # nulled out after the ComputeHostNotFound exception is raised.
  245. self.compute._resource_tracker = rt
  246. self.compute._update_available_resource_for_node(
  247. self.context,
  248. mock.sentinel.node,
  249. )
  250. rt.update_available_resource.assert_called_once_with(
  251. self.context,
  252. mock.sentinel.node,
  253. )
  254. self.assertTrue(log_mock.info.called)
  255. self.assertIsNone(self.compute._resource_tracker)
  256. @mock.patch.object(manager.ComputeManager, '_get_resource_tracker')
  257. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  258. 'delete_resource_provider')
  259. @mock.patch.object(manager.ComputeManager,
  260. '_update_available_resource_for_node')
  261. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  262. @mock.patch.object(manager.ComputeManager, '_get_compute_nodes_in_db')
  263. def test_update_available_resource(self, get_db_nodes, get_avail_nodes,
  264. update_mock, del_rp_mock, mock_get_rt):
  265. db_nodes = [self._make_compute_node('node%s' % i, i)
  266. for i in range(1, 5)]
  267. avail_nodes = set(['node2', 'node3', 'node4', 'node5'])
  268. avail_nodes_l = list(avail_nodes)
  269. get_db_nodes.return_value = db_nodes
  270. get_avail_nodes.return_value = avail_nodes
  271. self.compute.update_available_resource(self.context)
  272. get_db_nodes.assert_called_once_with(self.context, use_slave=True,
  273. startup=False)
  274. self.assertEqual(len(avail_nodes_l), update_mock.call_count)
  275. update_mock.assert_has_calls(
  276. [mock.call(self.context, node) for node in avail_nodes_l])
  277. # First node in set should have been removed from DB
  278. for db_node in db_nodes:
  279. if db_node.hypervisor_hostname == 'node1':
  280. db_node.destroy.assert_called_once_with()
  281. del_rp_mock.assert_called_once_with(self.context, db_node,
  282. cascade=True)
  283. mock_get_rt.return_value.remove_node.assert_called_once_with(
  284. 'node1')
  285. else:
  286. self.assertFalse(db_node.destroy.called)
  287. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  288. 'delete_resource_provider')
  289. @mock.patch.object(manager.ComputeManager,
  290. '_update_available_resource_for_node')
  291. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  292. @mock.patch.object(manager.ComputeManager, '_get_compute_nodes_in_db')
  293. def test_update_available_resource_not_ready(self, get_db_nodes,
  294. get_avail_nodes,
  295. update_mock,
  296. del_rp_mock):
  297. db_nodes = [self._make_compute_node('node1', 1)]
  298. get_db_nodes.return_value = db_nodes
  299. get_avail_nodes.side_effect = exception.VirtDriverNotReady
  300. self.compute.update_available_resource(self.context)
  301. # these shouldn't get processed on VirtDriverNotReady
  302. update_mock.assert_not_called()
  303. del_rp_mock.assert_not_called()
  304. @mock.patch('nova.context.get_admin_context')
  305. def test_pre_start_hook(self, get_admin_context):
  306. """Very simple test just to make sure update_available_resource is
  307. called as expected.
  308. """
  309. with mock.patch.object(
  310. self.compute, 'update_available_resource') as update_res:
  311. self.compute.pre_start_hook()
  312. update_res.assert_called_once_with(
  313. get_admin_context.return_value, startup=True)
  314. @mock.patch.object(objects.ComputeNodeList, 'get_all_by_host',
  315. side_effect=exception.NotFound)
  316. @mock.patch('nova.compute.manager.LOG')
  317. def test_get_compute_nodes_in_db_on_startup(self, mock_log,
  318. get_all_by_host):
  319. """Tests to make sure we only log a warning when we do not find a
  320. compute node on startup since this may be expected.
  321. """
  322. self.assertEqual([], self.compute._get_compute_nodes_in_db(
  323. self.context, startup=True))
  324. get_all_by_host.assert_called_once_with(
  325. self.context, self.compute.host, use_slave=False)
  326. self.assertTrue(mock_log.warning.called)
  327. self.assertFalse(mock_log.error.called)
  328. def _trusted_certs_setup_instance(self, include_trusted_certs=True):
  329. instance = fake_instance.fake_instance_obj(self.context)
  330. if include_trusted_certs:
  331. instance.trusted_certs = objects.trusted_certs.TrustedCerts(
  332. ids=['fake-trusted-cert-1', 'fake-trusted-cert-2'])
  333. else:
  334. instance.trusted_certs = None
  335. return instance
  336. def test_check_trusted_certs_provided_no_support(self):
  337. instance = self._trusted_certs_setup_instance()
  338. with mock.patch.dict(self.compute.driver.capabilities,
  339. supports_trusted_certs=False):
  340. self.assertRaises(exception.BuildAbortException,
  341. self.compute._check_trusted_certs,
  342. instance)
  343. def test_check_trusted_certs_not_provided_no_support(self):
  344. instance = self._trusted_certs_setup_instance(
  345. include_trusted_certs=False)
  346. with mock.patch.dict(self.compute.driver.capabilities,
  347. supports_trusted_certs=False):
  348. self.compute._check_trusted_certs(instance)
  349. def test_check_trusted_certs_provided_support(self):
  350. instance = self._trusted_certs_setup_instance()
  351. with mock.patch.dict(self.compute.driver.capabilities,
  352. supports_trusted_certs=True):
  353. self.compute._check_trusted_certs(instance)
  354. def test_check_device_tagging_no_tagging(self):
  355. bdms = objects.BlockDeviceMappingList(objects=[
  356. objects.BlockDeviceMapping(source_type='volume',
  357. destination_type='volume',
  358. instance_uuid=uuids.instance)])
  359. net_req = net_req_obj.NetworkRequest(tag=None)
  360. net_req_list = net_req_obj.NetworkRequestList(objects=[net_req])
  361. with mock.patch.dict(self.compute.driver.capabilities,
  362. supports_device_tagging=False):
  363. self.compute._check_device_tagging(net_req_list, bdms)
  364. def test_check_device_tagging_no_networks(self):
  365. bdms = objects.BlockDeviceMappingList(objects=[
  366. objects.BlockDeviceMapping(source_type='volume',
  367. destination_type='volume',
  368. instance_uuid=uuids.instance)])
  369. with mock.patch.dict(self.compute.driver.capabilities,
  370. supports_device_tagging=False):
  371. self.compute._check_device_tagging(None, bdms)
  372. def test_check_device_tagging_tagged_net_req_no_virt_support(self):
  373. bdms = objects.BlockDeviceMappingList(objects=[
  374. objects.BlockDeviceMapping(source_type='volume',
  375. destination_type='volume',
  376. instance_uuid=uuids.instance)])
  377. net_req = net_req_obj.NetworkRequest(port_id=uuids.bar, tag='foo')
  378. net_req_list = net_req_obj.NetworkRequestList(objects=[net_req])
  379. with mock.patch.dict(self.compute.driver.capabilities,
  380. supports_device_tagging=False):
  381. self.assertRaises(exception.BuildAbortException,
  382. self.compute._check_device_tagging,
  383. net_req_list, bdms)
  384. def test_check_device_tagging_tagged_bdm_no_driver_support(self):
  385. bdms = objects.BlockDeviceMappingList(objects=[
  386. objects.BlockDeviceMapping(source_type='volume',
  387. destination_type='volume',
  388. tag='foo',
  389. instance_uuid=uuids.instance)])
  390. with mock.patch.dict(self.compute.driver.capabilities,
  391. supports_device_tagging=False):
  392. self.assertRaises(exception.BuildAbortException,
  393. self.compute._check_device_tagging,
  394. None, bdms)
  395. def test_check_device_tagging_tagged_bdm_no_driver_support_declared(self):
  396. bdms = objects.BlockDeviceMappingList(objects=[
  397. objects.BlockDeviceMapping(source_type='volume',
  398. destination_type='volume',
  399. tag='foo',
  400. instance_uuid=uuids.instance)])
  401. with mock.patch.dict(self.compute.driver.capabilities):
  402. self.compute.driver.capabilities.pop('supports_device_tagging',
  403. None)
  404. self.assertRaises(exception.BuildAbortException,
  405. self.compute._check_device_tagging,
  406. None, bdms)
  407. def test_check_device_tagging_tagged_bdm_with_driver_support(self):
  408. bdms = objects.BlockDeviceMappingList(objects=[
  409. objects.BlockDeviceMapping(source_type='volume',
  410. destination_type='volume',
  411. tag='foo',
  412. instance_uuid=uuids.instance)])
  413. net_req = net_req_obj.NetworkRequest(network_id=uuids.bar)
  414. net_req_list = net_req_obj.NetworkRequestList(objects=[net_req])
  415. with mock.patch.dict(self.compute.driver.capabilities,
  416. supports_device_tagging=True):
  417. self.compute._check_device_tagging(net_req_list, bdms)
  418. def test_check_device_tagging_tagged_net_req_with_driver_support(self):
  419. bdms = objects.BlockDeviceMappingList(objects=[
  420. objects.BlockDeviceMapping(source_type='volume',
  421. destination_type='volume',
  422. instance_uuid=uuids.instance)])
  423. net_req = net_req_obj.NetworkRequest(network_id=uuids.bar, tag='foo')
  424. net_req_list = net_req_obj.NetworkRequestList(objects=[net_req])
  425. with mock.patch.dict(self.compute.driver.capabilities,
  426. supports_device_tagging=True):
  427. self.compute._check_device_tagging(net_req_list, bdms)
  428. @mock.patch.object(objects.BlockDeviceMapping, 'create')
  429. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid',
  430. return_value=objects.BlockDeviceMappingList())
  431. def test_reserve_block_device_name_with_tag(self, mock_get, mock_create):
  432. instance = fake_instance.fake_instance_obj(self.context)
  433. with test.nested(
  434. mock.patch.object(self.compute,
  435. '_get_device_name_for_instance',
  436. return_value='/dev/vda'),
  437. mock.patch.dict(self.compute.driver.capabilities,
  438. supports_tagged_attach_volume=True)):
  439. bdm = self.compute.reserve_block_device_name(
  440. self.context, instance, None, None, None, None, 'foo',
  441. False)
  442. self.assertEqual('foo', bdm.tag)
  443. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  444. def test_reserve_block_device_name_raises(self, _):
  445. with mock.patch.dict(self.compute.driver.capabilities,
  446. supports_tagged_attach_volume=False):
  447. self.assertRaises(exception.VolumeTaggedAttachNotSupported,
  448. self.compute.reserve_block_device_name,
  449. self.context,
  450. fake_instance.fake_instance_obj(self.context),
  451. 'fake_device', 'fake_volume_id', 'fake_disk_bus',
  452. 'fake_device_type', 'foo', False)
  453. @mock.patch.object(objects.BlockDeviceMapping, 'create')
  454. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid',
  455. return_value=objects.BlockDeviceMappingList())
  456. def test_reserve_block_device_name_multiattach(self, mock_get,
  457. mock_create):
  458. """Tests the case that multiattach=True and the driver supports it."""
  459. instance = fake_instance.fake_instance_obj(self.context)
  460. with test.nested(
  461. mock.patch.object(self.compute,
  462. '_get_device_name_for_instance',
  463. return_value='/dev/vda'),
  464. mock.patch.dict(self.compute.driver.capabilities,
  465. supports_multiattach=True)):
  466. self.compute.reserve_block_device_name(
  467. self.context, instance, device=None, volume_id=uuids.volume_id,
  468. disk_bus=None, device_type=None, tag=None, multiattach=True)
  469. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  470. def test_reserve_block_device_name_multiattach_raises(self, _):
  471. with mock.patch.dict(self.compute.driver.capabilities,
  472. supports_multiattach=False):
  473. self.assertRaises(exception.MultiattachNotSupportedByVirtDriver,
  474. self.compute.reserve_block_device_name,
  475. self.context,
  476. fake_instance.fake_instance_obj(self.context),
  477. 'fake_device', 'fake_volume_id', 'fake_disk_bus',
  478. 'fake_device_type', tag=None, multiattach=True)
  479. @mock.patch.object(objects.Instance, 'save')
  480. @mock.patch.object(time, 'sleep')
  481. def test_allocate_network_succeeds_after_retries(
  482. self, mock_sleep, mock_save):
  483. self.flags(network_allocate_retries=8)
  484. instance = fake_instance.fake_instance_obj(
  485. self.context, expected_attrs=['system_metadata'])
  486. is_vpn = 'fake-is-vpn'
  487. req_networks = objects.NetworkRequestList(
  488. objects=[objects.NetworkRequest(network_id='fake')])
  489. macs = 'fake-macs'
  490. sec_groups = 'fake-sec-groups'
  491. final_result = 'meow'
  492. expected_sleep_times = [mock.call(t) for t in
  493. (1, 2, 4, 8, 16, 30, 30)]
  494. with mock.patch.object(
  495. self.compute.network_api, 'allocate_for_instance',
  496. side_effect=[test.TestingException()] * 7 + [final_result]):
  497. res = self.compute._allocate_network_async(self.context, instance,
  498. req_networks,
  499. macs,
  500. sec_groups,
  501. is_vpn)
  502. self.assertEqual(7, mock_sleep.call_count)
  503. mock_sleep.assert_has_calls(expected_sleep_times)
  504. self.assertEqual(final_result, res)
  505. # Ensure save is not called in while allocating networks, the instance
  506. # is saved after the allocation.
  507. self.assertFalse(mock_save.called)
  508. self.assertEqual('True', instance.system_metadata['network_allocated'])
  509. def test_allocate_network_fails(self):
  510. self.flags(network_allocate_retries=0)
  511. instance = {}
  512. is_vpn = 'fake-is-vpn'
  513. req_networks = objects.NetworkRequestList(
  514. objects=[objects.NetworkRequest(network_id='fake')])
  515. macs = 'fake-macs'
  516. sec_groups = 'fake-sec-groups'
  517. with mock.patch.object(
  518. self.compute.network_api, 'allocate_for_instance',
  519. side_effect=test.TestingException) as mock_allocate:
  520. self.assertRaises(test.TestingException,
  521. self.compute._allocate_network_async,
  522. self.context, instance, req_networks, macs,
  523. sec_groups, is_vpn)
  524. mock_allocate.assert_called_once_with(
  525. self.context, instance, vpn=is_vpn,
  526. requested_networks=req_networks, macs=macs,
  527. security_groups=sec_groups,
  528. bind_host_id=instance.get('host'))
  529. @mock.patch.object(manager.ComputeManager, '_instance_update')
  530. @mock.patch.object(time, 'sleep')
  531. def test_allocate_network_with_conf_value_is_one(
  532. self, sleep, _instance_update):
  533. self.flags(network_allocate_retries=1)
  534. instance = fake_instance.fake_instance_obj(
  535. self.context, expected_attrs=['system_metadata'])
  536. is_vpn = 'fake-is-vpn'
  537. req_networks = objects.NetworkRequestList(
  538. objects=[objects.NetworkRequest(network_id='fake')])
  539. macs = 'fake-macs'
  540. sec_groups = 'fake-sec-groups'
  541. final_result = 'zhangtralon'
  542. with mock.patch.object(self.compute.network_api,
  543. 'allocate_for_instance',
  544. side_effect = [test.TestingException(),
  545. final_result]):
  546. res = self.compute._allocate_network_async(self.context, instance,
  547. req_networks,
  548. macs,
  549. sec_groups,
  550. is_vpn)
  551. self.assertEqual(final_result, res)
  552. self.assertEqual(1, sleep.call_count)
  553. def test_allocate_network_skip_for_no_allocate(self):
  554. # Ensures that we don't do anything if requested_networks has 'none'
  555. # for the network_id.
  556. req_networks = objects.NetworkRequestList(
  557. objects=[objects.NetworkRequest(network_id='none')])
  558. nwinfo = self.compute._allocate_network_async(
  559. self.context, mock.sentinel.instance, req_networks, macs=None,
  560. security_groups=['default'], is_vpn=False)
  561. self.assertEqual(0, len(nwinfo))
  562. @mock.patch('nova.compute.manager.ComputeManager.'
  563. '_do_build_and_run_instance')
  564. def _test_max_concurrent_builds(self, mock_dbari):
  565. with mock.patch.object(self.compute,
  566. '_build_semaphore') as mock_sem:
  567. instance = objects.Instance(uuid=uuidutils.generate_uuid())
  568. for i in (1, 2, 3):
  569. self.compute.build_and_run_instance(self.context, instance,
  570. mock.sentinel.image,
  571. mock.sentinel.request_spec,
  572. {})
  573. self.assertEqual(3, mock_sem.__enter__.call_count)
  574. def test_max_concurrent_builds_limited(self):
  575. self.flags(max_concurrent_builds=2)
  576. self._test_max_concurrent_builds()
  577. def test_max_concurrent_builds_unlimited(self):
  578. self.flags(max_concurrent_builds=0)
  579. self._test_max_concurrent_builds()
  580. def test_max_concurrent_builds_semaphore_limited(self):
  581. self.flags(max_concurrent_builds=123)
  582. self.assertEqual(123,
  583. manager.ComputeManager()._build_semaphore.balance)
  584. def test_max_concurrent_builds_semaphore_unlimited(self):
  585. self.flags(max_concurrent_builds=0)
  586. compute = manager.ComputeManager()
  587. self.assertEqual(0, compute._build_semaphore.balance)
  588. self.assertIsInstance(compute._build_semaphore,
  589. compute_utils.UnlimitedSemaphore)
  590. def test_nil_out_inst_obj_host_and_node_sets_nil(self):
  591. instance = fake_instance.fake_instance_obj(self.context,
  592. uuid=uuids.instance,
  593. host='foo-host',
  594. node='foo-node')
  595. self.assertIsNotNone(instance.host)
  596. self.assertIsNotNone(instance.node)
  597. self.compute._nil_out_instance_obj_host_and_node(instance)
  598. self.assertIsNone(instance.host)
  599. self.assertIsNone(instance.node)
  600. def test_init_host(self):
  601. our_host = self.compute.host
  602. inst = fake_instance.fake_db_instance(
  603. vm_state=vm_states.ACTIVE,
  604. info_cache=dict(test_instance_info_cache.fake_info_cache,
  605. network_info=None),
  606. security_groups=None)
  607. startup_instances = [inst, inst, inst]
  608. def _make_instance_list(db_list):
  609. return instance_obj._make_instance_list(
  610. self.context, objects.InstanceList(), db_list, None)
  611. @mock.patch.object(manager.ComputeManager,
  612. '_error_out_instances_whose_build_was_interrupted')
  613. @mock.patch.object(fake_driver.FakeDriver, 'init_host')
  614. @mock.patch.object(fake_driver.FakeDriver, 'filter_defer_apply_on')
  615. @mock.patch.object(fake_driver.FakeDriver, 'filter_defer_apply_off')
  616. @mock.patch.object(objects.InstanceList, 'get_by_host')
  617. @mock.patch.object(context, 'get_admin_context')
  618. @mock.patch.object(manager.ComputeManager,
  619. '_destroy_evacuated_instances')
  620. @mock.patch.object(manager.ComputeManager, '_init_instance')
  621. @mock.patch.object(self.compute, '_update_scheduler_instance_info')
  622. def _do_mock_calls(mock_update_scheduler, mock_inst_init,
  623. mock_destroy, mock_admin_ctxt, mock_host_get,
  624. mock_filter_off, mock_filter_on, mock_init_host,
  625. mock_error_interrupted, defer_iptables_apply):
  626. mock_admin_ctxt.return_value = self.context
  627. inst_list = _make_instance_list(startup_instances)
  628. mock_host_get.return_value = inst_list
  629. self.compute.init_host()
  630. if defer_iptables_apply:
  631. self.assertTrue(mock_filter_on.called)
  632. mock_destroy.assert_called_once_with(self.context)
  633. mock_inst_init.assert_has_calls(
  634. [mock.call(self.context, inst_list[0]),
  635. mock.call(self.context, inst_list[1]),
  636. mock.call(self.context, inst_list[2])])
  637. if defer_iptables_apply:
  638. self.assertTrue(mock_filter_off.called)
  639. mock_init_host.assert_called_once_with(host=our_host)
  640. mock_host_get.assert_called_once_with(self.context, our_host,
  641. expected_attrs=['info_cache', 'metadata'])
  642. mock_update_scheduler.assert_called_once_with(
  643. self.context, inst_list)
  644. mock_error_interrupted.assert_called_once_with(
  645. self.context, {inst.uuid for inst in inst_list})
  646. # Test with defer_iptables_apply
  647. self.flags(defer_iptables_apply=True)
  648. _do_mock_calls(defer_iptables_apply=True)
  649. # Test without defer_iptables_apply
  650. self.flags(defer_iptables_apply=False)
  651. _do_mock_calls(defer_iptables_apply=False)
  652. @mock.patch('nova.compute.manager.ComputeManager.'
  653. '_error_out_instances_whose_build_was_interrupted')
  654. @mock.patch('nova.objects.InstanceList.get_by_host',
  655. return_value=objects.InstanceList())
  656. @mock.patch('nova.compute.manager.ComputeManager.'
  657. '_destroy_evacuated_instances')
  658. @mock.patch('nova.compute.manager.ComputeManager._init_instance',
  659. mock.NonCallableMock())
  660. @mock.patch('nova.compute.manager.ComputeManager.'
  661. '_update_scheduler_instance_info', mock.NonCallableMock())
  662. def test_init_host_no_instances(self, mock_destroy_evac_instances,
  663. mock_get_by_host, mock_error_interrupted):
  664. """Tests the case that init_host runs and there are no instances
  665. on this host yet (it's brand new). Uses NonCallableMock for the
  666. methods we assert should not be called.
  667. """
  668. self.compute.init_host()
  669. mock_error_interrupted.assert_called_once_with(mock.ANY, set())
  670. @mock.patch('nova.objects.InstanceList')
  671. @mock.patch('nova.objects.MigrationList.get_by_filters')
  672. def test_cleanup_host(self, mock_miglist_get, mock_instance_list):
  673. # just testing whether the cleanup_host method
  674. # when fired will invoke the underlying driver's
  675. # equivalent method.
  676. mock_miglist_get.return_value = []
  677. mock_instance_list.get_by_host.return_value = []
  678. with mock.patch.object(self.compute, 'driver') as mock_driver:
  679. self.compute.init_host()
  680. mock_driver.init_host.assert_called_once_with(host='fake-mini')
  681. self.compute.cleanup_host()
  682. # register_event_listener is called on startup (init_host) and
  683. # in cleanup_host
  684. mock_driver.register_event_listener.assert_has_calls([
  685. mock.call(self.compute.handle_events), mock.call(None)])
  686. mock_driver.cleanup_host.assert_called_once_with(host='fake-mini')
  687. def test_cleanup_live_migrations_in_pool_with_record(self):
  688. fake_future = mock.MagicMock()
  689. fake_instance_uuid = uuids.instance
  690. fake_migration = objects.Migration(
  691. uuid=uuids.migration, instance_uuid=fake_instance_uuid)
  692. fake_migration.save = mock.MagicMock()
  693. self.compute._waiting_live_migrations[fake_instance_uuid] = (
  694. fake_migration, fake_future)
  695. with mock.patch.object(self.compute, '_live_migration_executor'
  696. ) as mock_migration_pool:
  697. self.compute._cleanup_live_migrations_in_pool()
  698. mock_migration_pool.shutdown.assert_called_once_with(wait=False)
  699. self.assertEqual('cancelled', fake_migration.status)
  700. fake_future.cancel.assert_called_once_with()
  701. self.assertEqual({}, self.compute._waiting_live_migrations)
  702. # test again with Future is None
  703. self.compute._waiting_live_migrations[fake_instance_uuid] = (
  704. None, None)
  705. self.compute._cleanup_live_migrations_in_pool()
  706. mock_migration_pool.shutdown.assert_called_with(wait=False)
  707. self.assertEqual(2, mock_migration_pool.shutdown.call_count)
  708. self.assertEqual({}, self.compute._waiting_live_migrations)
  709. def test_init_virt_events_disabled(self):
  710. self.flags(handle_virt_lifecycle_events=False, group='workarounds')
  711. with mock.patch.object(self.compute.driver,
  712. 'register_event_listener') as mock_register:
  713. self.compute.init_virt_events()
  714. self.assertFalse(mock_register.called)
  715. @mock.patch('nova.context.RequestContext.elevated')
  716. @mock.patch.object(manager.ComputeManager,
  717. '_error_out_instances_whose_build_was_interrupted')
  718. @mock.patch('nova.objects.ComputeNode.get_by_host_and_nodename')
  719. @mock.patch('nova.scheduler.utils.resources_from_flavor')
  720. @mock.patch.object(manager.ComputeManager, '_get_instances_on_driver')
  721. @mock.patch.object(manager.ComputeManager, 'init_virt_events')
  722. @mock.patch.object(context, 'get_admin_context')
  723. @mock.patch.object(objects.InstanceList, 'get_by_host')
  724. @mock.patch.object(fake_driver.FakeDriver, 'destroy')
  725. @mock.patch.object(fake_driver.FakeDriver, 'init_host')
  726. @mock.patch('nova.utils.temporary_mutation')
  727. @mock.patch('nova.objects.MigrationList.get_by_filters')
  728. @mock.patch('nova.objects.Migration.save')
  729. def test_init_host_with_evacuated_instance(self, mock_save, mock_mig_get,
  730. mock_temp_mut, mock_init_host, mock_destroy, mock_host_get,
  731. mock_admin_ctxt, mock_init_virt, mock_get_inst, mock_resources,
  732. mock_get_node, mock_error_interrupted, mock_elevated):
  733. our_host = self.compute.host
  734. not_our_host = 'not-' + our_host
  735. read_deleted_context = self.context.elevated(read_deleted='yes')
  736. mock_elevated.return_value = read_deleted_context
  737. deleted_instance = fake_instance.fake_instance_obj(
  738. self.context, host=not_our_host, uuid=uuids.deleted_instance)
  739. migration = objects.Migration(instance_uuid=deleted_instance.uuid)
  740. migration.source_node = 'fake-node'
  741. mock_mig_get.return_value = [migration]
  742. mock_admin_ctxt.return_value = self.context
  743. mock_host_get.return_value = objects.InstanceList()
  744. our_node = objects.ComputeNode(host=our_host, uuid=uuids.our_node_uuid)
  745. mock_get_node.return_value = our_node
  746. mock_resources.return_value = mock.sentinel.my_resources
  747. # simulate failed instance
  748. mock_get_inst.return_value = [deleted_instance]
  749. with test.nested(
  750. mock.patch.object(
  751. self.compute.network_api, 'get_instance_nw_info',
  752. side_effect = exception.InstanceNotFound(
  753. instance_id=deleted_instance['uuid'])),
  754. mock.patch.object(
  755. self.compute.reportclient,
  756. 'remove_provider_from_instance_allocation')
  757. ) as (mock_get_net, mock_remove_allocation):
  758. self.compute.init_host()
  759. mock_remove_allocation.assert_called_once_with(
  760. self.context, deleted_instance.uuid, uuids.our_node_uuid,
  761. deleted_instance.user_id, deleted_instance.project_id,
  762. mock.sentinel.my_resources)
  763. mock_init_host.assert_called_once_with(host=our_host)
  764. mock_host_get.assert_called_once_with(self.context, our_host,
  765. expected_attrs=['info_cache', 'metadata'])
  766. mock_init_virt.assert_called_once_with()
  767. mock_temp_mut.assert_called_once_with(self.context, read_deleted='yes')
  768. mock_get_inst.assert_called_once_with(read_deleted_context)
  769. mock_get_net.assert_called_once_with(self.context, deleted_instance)
  770. # ensure driver.destroy is called so that driver may
  771. # clean up any dangling files
  772. mock_destroy.assert_called_once_with(self.context, deleted_instance,
  773. mock.ANY, mock.ANY, mock.ANY)
  774. mock_save.assert_called_once_with()
  775. mock_error_interrupted.assert_called_once_with(
  776. self.context, {deleted_instance.uuid})
  777. @mock.patch.object(manager.ComputeManager,
  778. '_error_out_instances_whose_build_was_interrupted')
  779. @mock.patch.object(context, 'get_admin_context')
  780. @mock.patch.object(objects.InstanceList, 'get_by_host')
  781. @mock.patch.object(fake_driver.FakeDriver, 'init_host')
  782. @mock.patch('nova.compute.manager.ComputeManager._init_instance')
  783. @mock.patch('nova.compute.manager.ComputeManager.'
  784. '_destroy_evacuated_instances')
  785. def test_init_host_with_in_progress_evacuations(self, mock_destroy_evac,
  786. mock_init_instance, mock_init_host, mock_host_get,
  787. mock_admin_ctxt, mock_error_interrupted):
  788. """Assert that init_instance is not called for instances that are
  789. evacuating from the host during init_host.
  790. """
  791. active_instance = fake_instance.fake_instance_obj(
  792. self.context, host=self.compute.host, uuid=uuids.active_instance)
  793. evacuating_instance = fake_instance.fake_instance_obj(
  794. self.context, host=self.compute.host, uuid=uuids.evac_instance)
  795. instance_list = objects.InstanceList(self.context,
  796. objects=[active_instance, evacuating_instance])
  797. mock_host_get.return_value = instance_list
  798. mock_admin_ctxt.return_value = self.context
  799. mock_destroy_evac.return_value = {
  800. uuids.evac_instance: evacuating_instance
  801. }
  802. self.compute.init_host()
  803. mock_init_instance.assert_called_once_with(
  804. self.context, active_instance)
  805. mock_error_interrupted.assert_called_once_with(
  806. self.context, {active_instance.uuid, evacuating_instance.uuid})
  807. @mock.patch.object(objects.Instance, 'save')
  808. @mock.patch.object(objects.InstanceList, 'get_by_filters')
  809. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  810. 'get_allocations_for_resource_provider')
  811. @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
  812. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  813. def test_init_host_with_interrupted_instance_build(
  814. self, mock_get_nodes, mock_get_by_host_and_node,
  815. mock_get_allocations, mock_get_instances, mock_instance_save):
  816. mock_get_nodes.return_value = ['fake-node']
  817. mock_get_by_host_and_node.return_value = objects.ComputeNode(
  818. host=self.compute.host, uuid=uuids.cn_uuid)
  819. active_instance = fake_instance.fake_instance_obj(
  820. self.context, host=self.compute.host, uuid=uuids.active_instance)
  821. evacuating_instance = fake_instance.fake_instance_obj(
  822. self.context, host=self.compute.host, uuid=uuids.evac_instance)
  823. interrupted_instance = fake_instance.fake_instance_obj(
  824. self.context, host=None, uuid=uuids.interrupted_instance,
  825. vm_state=vm_states.BUILDING)
  826. # we have 3 different instances. We need consumers for each instance
  827. # in placement and an extra consumer that is not an instance
  828. allocations = {
  829. uuids.active_instance: "fake-resources-active",
  830. uuids.evac_instance: "fake-resources-evacuating",
  831. uuids.interrupted_instance: "fake-resources-interrupted",
  832. uuids.not_an_instance: "fake-resources-not-an-instance",
  833. }
  834. mock_get_allocations.return_value = allocations
  835. # get is called with a uuid filter containing interrupted_instance,
  836. # error_instance, and not_an_instance but it will only return the
  837. # interrupted_instance as the error_instance is not in building state
  838. # and not_an_instance does not match with any instance in the db.
  839. mock_get_instances.return_value = objects.InstanceList(
  840. self.context, objects=[interrupted_instance])
  841. # interrupted_instance and error_instance is not in the list passed in
  842. # because it is not assigned to the compute and therefore not processed
  843. # by init_host and init_instance
  844. self.compute._error_out_instances_whose_build_was_interrupted(
  845. self.context,
  846. {inst.uuid for inst in [active_instance, evacuating_instance]})
  847. mock_get_by_host_and_node.assert_called_once_with(
  848. self.context, self.compute.host, 'fake-node')
  849. mock_get_allocations.assert_called_once_with(
  850. self.context, uuids.cn_uuid)
  851. mock_get_instances.assert_called_once_with(
  852. self.context,
  853. {'vm_state': 'building',
  854. 'uuid': {uuids.interrupted_instance, uuids.not_an_instance}
  855. },
  856. expected_attrs=[])
  857. # this is expected to be called only once for interrupted_instance
  858. mock_instance_save.assert_called_once_with()
  859. self.assertEqual(vm_states.ERROR, interrupted_instance.vm_state)
  860. @mock.patch.object(manager.LOG, 'error')
  861. @mock.patch.object(objects.Instance, 'save')
  862. @mock.patch.object(objects.InstanceList, 'get_by_filters')
  863. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  864. 'get_allocations_for_resource_provider')
  865. @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
  866. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  867. def test_init_host_with_interrupted_instance_build_empty_compute(
  868. self, mock_get_nodes, mock_get_by_host_and_node,
  869. mock_get_allocations, mock_get_instances, mock_instance_save,
  870. mock_log):
  871. mock_get_nodes.return_value = ['fake-node']
  872. mock_get_by_host_and_node.return_value = objects.ComputeNode(
  873. host=self.compute.host, uuid=uuids.cn_uuid)
  874. # no instances on the host so no allocations in placement
  875. allocations = {}
  876. mock_get_allocations.return_value = allocations
  877. mock_get_instances.return_value = objects.InstanceList(
  878. self.context, objects=[])
  879. self.compute._error_out_instances_whose_build_was_interrupted(
  880. self.context, set())
  881. mock_get_by_host_and_node.assert_called_once_with(
  882. self.context, self.compute.host, 'fake-node')
  883. mock_get_allocations.assert_called_once_with(
  884. self.context, uuids.cn_uuid)
  885. mock_get_instances.assert_not_called()
  886. mock_instance_save.assert_not_called()
  887. mock_log.assert_not_called()
  888. @mock.patch.object(manager.LOG, 'error')
  889. @mock.patch.object(objects.Instance, 'save')
  890. @mock.patch.object(objects.InstanceList, 'get_by_filters')
  891. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  892. 'get_allocations_for_resource_provider')
  893. @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
  894. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  895. def test_init_host_with_interrupted_instance_build_placement_error(
  896. self, mock_get_nodes, mock_get_by_host_and_node,
  897. mock_get_allocations, mock_get_instances, mock_instance_save,
  898. mock_log):
  899. mock_get_nodes.return_value = ['fake-node']
  900. mock_get_by_host_and_node.return_value = objects.ComputeNode(
  901. host=self.compute.host, uuid=uuids.cn_uuid)
  902. # get_allocations_for_resource_provider returns None if placement
  903. # returns an error
  904. allocations = None
  905. mock_get_allocations.return_value = allocations
  906. mock_get_instances.return_value = objects.InstanceList(
  907. self.context, objects=[])
  908. self.compute._error_out_instances_whose_build_was_interrupted(
  909. self.context, set())
  910. mock_get_by_host_and_node.assert_called_once_with(
  911. self.context, self.compute.host, 'fake-node')
  912. mock_get_allocations.assert_called_once_with(
  913. self.context, uuids.cn_uuid)
  914. mock_get_instances.assert_not_called()
  915. mock_instance_save.assert_not_called()
  916. mock_log.assert_called_once_with(
  917. 'Could not retrieve compute node resource provider %s and '
  918. 'therefore unable to error out any instances stuck in '
  919. 'BUILDING state.', uuids.cn_uuid)
  920. @mock.patch.object(manager.LOG, 'warning')
  921. @mock.patch.object(
  922. fake_driver.FakeDriver, 'get_available_nodes',
  923. side_effect=exception.VirtDriverNotReady)
  924. def test_init_host_with_interrupted_instance_build_driver_not_ready(
  925. self, mock_get_nodes, mock_log_warning):
  926. self.compute._error_out_instances_whose_build_was_interrupted(
  927. self.context, set())
  928. mock_log_warning.assert_called_once_with(
  929. "Virt driver is not ready. Therefore unable to error out any "
  930. "instances stuck in BUILDING state on this node. If this is the "
  931. "first time this service is starting on this host, then you can "
  932. "ignore this warning.")
  933. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  934. 'get_allocations_for_resource_provider')
  935. @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
  936. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  937. def test_init_host_with_interrupted_instance_build_compute_node_not_found(
  938. self, mock_get_nodes, mock_get_by_host_and_node,
  939. mock_get_allocations):
  940. mock_get_nodes.return_value = ['fake-node1', 'fake-node2']
  941. mock_get_by_host_and_node.side_effect = [
  942. exception.ComputeHostNotFound(host='fake-node1'),
  943. objects.ComputeNode(host=self.compute.host, uuid=uuids.cn_uuid)]
  944. self.compute._error_out_instances_whose_build_was_interrupted(
  945. self.context, set())
  946. # check that nova skip the node that is not found in the db and
  947. # continue with the next
  948. mock_get_by_host_and_node.assert_has_calls(
  949. [
  950. mock.call(self.context, self.compute.host, 'fake-node1'),
  951. mock.call(self.context, self.compute.host, 'fake-node2'),
  952. ]
  953. )
  954. # placement only queried for the existing compute
  955. mock_get_allocations.assert_called_once_with(
  956. self.context, uuids.cn_uuid)
  957. @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
  958. 'get_allocations_for_resource_provider')
  959. @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
  960. @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
  961. def test_init_host_with_interrupted_instance_build_compute_rp_not_found(
  962. self, mock_get_nodes, mock_get_by_host_and_node,
  963. mock_get_allocations):
  964. mock_get_nodes.return_value = ['fake-node1', 'fake-node2']
  965. mock_get_by_host_and_node.side_effect = [
  966. objects.ComputeNode(host=self.compute.host, uuid=uuids.cn1_uuid),
  967. objects.ComputeNode(host=self.compute.host, uuid=uuids.cn2_uuid),
  968. ]
  969. mock_get_allocations.side_effect = [
  970. {},
  971. {uuids.active_instance: "fake-resources"}
  972. ]
  973. self.compute._error_out_instances_whose_build_was_interrupted(
  974. self.context, {uuids.active_instance})
  975. # check that nova skip the node that is not found in placement and
  976. # continue with the next
  977. mock_get_allocations.assert_has_calls(
  978. [
  979. mock.call(self.context, uuids.cn1_uuid),
  980. mock.call(self.context, uuids.cn2_uuid),
  981. ]
  982. )
  983. def test_init_instance_with_binding_failed_vif_type(self):
  984. # this instance will plug a 'binding_failed' vif
  985. instance = fake_instance.fake_instance_obj(
  986. self.context,
  987. uuid=uuids.instance,
  988. info_cache=None,
  989. power_state=power_state.RUNNING,
  990. vm_state=vm_states.ACTIVE,
  991. task_state=None,
  992. host=self.compute.host,
  993. expected_attrs=['info_cache'])
  994. with test.nested(
  995. mock.patch.object(context, 'get_admin_context',
  996. return_value=self.context),
  997. mock.patch.object(objects.Instance, 'get_network_info',
  998. return_value=network_model.NetworkInfo()),
  999. mock.patch.object(self.compute.driver, 'plug_vifs',
  1000. side_effect=exception.VirtualInterfacePlugException(
  1001. "Unexpected vif_type=binding_failed")),
  1002. mock.patch.object(self.compute, '_set_instance_obj_error_state')
  1003. ) as (get_admin_context, get_nw_info, plug_vifs, set_error_state):
  1004. self.compute._init_instance(self.context, instance)
  1005. set_error_state.assert_called_once_with(self.context, instance)
  1006. def test__get_power_state_InstanceNotFound(self):
  1007. instance = fake_instance.fake_instance_obj(
  1008. self.context,
  1009. power_state=power_state.RUNNING)
  1010. with mock.patch.object(self.compute.driver,
  1011. 'get_info',
  1012. side_effect=exception.InstanceNotFound(instance_id=1)):
  1013. self.assertEqual(self.compute._get_power_state(self.context,
  1014. instance),
  1015. power_state.NOSTATE)
  1016. def test__get_power_state_NotFound(self):
  1017. instance = fake_instance.fake_instance_obj(
  1018. self.context,
  1019. power_state=power_state.RUNNING)
  1020. with mock.patch.object(self.compute.driver,
  1021. 'get_info',
  1022. side_effect=exception.NotFound()):
  1023. self.assertRaises(exception.NotFound,
  1024. self.compute._get_power_state,
  1025. self.context, instance)
  1026. @mock.patch.object(manager.ComputeManager, '_get_power_state')
  1027. @mock.patch.object(fake_driver.FakeDriver, 'plug_vifs')
  1028. @mock.patch.object(fake_driver.FakeDriver, 'resume_state_on_host_boot')
  1029. @mock.patch.object(manager.ComputeManager,
  1030. '_get_instance_block_device_info')
  1031. @mock.patch.object(manager.ComputeManager, '_set_instance_obj_error_state')
  1032. def test_init_instance_failed_resume_sets_error(self, mock_set_inst,
  1033. mock_get_inst, mock_resume, mock_plug, mock_get_power):
  1034. instance = fake_instance.fake_instance_obj(
  1035. self.context,
  1036. uuid=uuids.instance,
  1037. info_cache=None,
  1038. power_state=power_state.RUNNING,
  1039. vm_state=vm_states.ACTIVE,
  1040. task_state=None,
  1041. host=self.compute.host,
  1042. expected_attrs=['info_cache'])
  1043. self.flags(resume_guests_state_on_host_boot=True)
  1044. mock_get_power.side_effect = (power_state.SHUTDOWN,
  1045. power_state.SHUTDOWN)
  1046. mock_get_inst.return_value = 'fake-bdm'
  1047. mock_resume.side_effect = test.TestingException
  1048. self.compute._init_instance('fake-context', instance)
  1049. mock_get_power.assert_has_calls([mock.call(mock.ANY, instance),
  1050. mock.call(mock.ANY, instance)])
  1051. mock_plug.assert_called_once_with(instance, mock.ANY)
  1052. mock_get_inst.assert_called_once_with(mock.ANY, instance)
  1053. mock_resume.assert_called_once_with(mock.ANY, instance, mock.ANY,
  1054. 'fake-bdm')
  1055. mock_set_inst.assert_called_once_with(mock.ANY, instance)
  1056. @mock.patch.object(objects.BlockDeviceMapping, 'destroy')
  1057. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
  1058. @mock.patch.object(objects.Instance, 'destroy')
  1059. @mock.patch.object(objects.Instance, 'obj_load_attr')
  1060. @mock.patch.object(objects.quotas, 'ids_from_instance')
  1061. def test_init_instance_complete_partial_deletion(
  1062. self, mock_ids_from_instance,
  1063. mock_inst_destroy, mock_obj_load_attr, mock_get_by_instance_uuid,
  1064. mock_bdm_destroy):
  1065. """Test to complete deletion for instances in DELETED status but not
  1066. marked as deleted in the DB
  1067. """
  1068. instance = fake_instance.fake_instance_obj(
  1069. self.context,
  1070. project_id=fakes.FAKE_PROJECT_ID,
  1071. uuid=uuids.instance,
  1072. vcpus=1,
  1073. memory_mb=64,
  1074. power_state=power_state.SHUTDOWN,
  1075. vm_state=vm_states.DELETED,
  1076. host=self.compute.host,
  1077. task_state=None,
  1078. deleted=False,
  1079. deleted_at=None,
  1080. metadata={},
  1081. system_metadata={},
  1082. expected_attrs=['metadata', 'system_metadata'])
  1083. # Make sure instance vm_state is marked as 'DELETED' but instance is
  1084. # not destroyed from db.
  1085. self.assertEqual(vm_states.DELETED, instance.vm_state)
  1086. self.assertFalse(instance.deleted)
  1087. def fake_inst_destroy():
  1088. instance.deleted = True
  1089. instance.deleted_at = timeutils.utcnow()
  1090. mock_ids_from_instance.return_value = (instance.project_id,
  1091. instance.user_id)
  1092. mock_inst_destroy.side_effect = fake_inst_destroy()
  1093. self.compute._init_instance(self.context, instance)
  1094. # Make sure that instance.destroy method was called and
  1095. # instance was deleted from db.
  1096. self.assertNotEqual(0, instance.deleted)
  1097. @mock.patch('nova.compute.manager.LOG')
  1098. def test_init_instance_complete_partial_deletion_raises_exception(
  1099. self, mock_log):
  1100. instance = fake_instance.fake_instance_obj(
  1101. self.context,
  1102. project_id=fakes.FAKE_PROJECT_ID,
  1103. uuid=uuids.instance,
  1104. vcpus=1,
  1105. memory_mb=64,
  1106. power_state=power_state.SHUTDOWN,
  1107. vm_state=vm_states.DELETED,
  1108. host=self.compute.host,
  1109. task_state=None,
  1110. deleted=False,
  1111. deleted_at=None,
  1112. metadata={},
  1113. system_metadata={},
  1114. expected_attrs=['metadata', 'system_metadata'])
  1115. with mock.patch.object(self.compute,
  1116. '_complete_partial_deletion') as mock_deletion:
  1117. mock_deletion.side_effect = test.TestingException()
  1118. self.compute._init_instance(self, instance)
  1119. msg = u'Failed to complete a deletion'
  1120. mock_log.exception.assert_called_once_with(msg, instance=instance)
  1121. def test_init_instance_stuck_in_deleting(self):
  1122. instance = fake_instance.fake_instance_obj(
  1123. self.context,
  1124. project_id=fakes.FAKE_PROJECT_ID,
  1125. uuid=uuids.instance,
  1126. vcpus=1,
  1127. memory_mb=64,
  1128. power_state=power_state.RUNNING,
  1129. vm_state=vm_states.ACTIVE,
  1130. host=self.compute.host,
  1131. task_state=task_states.DELETING)
  1132. bdms = []
  1133. with test.nested(
  1134. mock.patch.object(objects.BlockDeviceMappingList,
  1135. 'get_by_instance_uuid',
  1136. return_value=bdms),
  1137. mock.patch.object(self.compute, '_delete_instance'),
  1138. mock.patch.object(instance, 'obj_load_attr')
  1139. ) as (mock_get, mock_delete, mock_load):
  1140. self.compute._init_instance(self.context, instance)
  1141. mock_get.assert_called_once_with(self.context, instance.uuid)
  1142. mock_delete.assert_called_once_with(self.context, instance,
  1143. bdms)
  1144. @mock.patch.object(objects.Instance, 'get_by_uuid')
  1145. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
  1146. def test_init_instance_stuck_in_deleting_raises_exception(
  1147. self, mock_get_by_instance_uuid, mock_get_by_uuid):
  1148. instance = fake_instance.fake_instance_obj(
  1149. self.context,
  1150. project_id=fakes.FAKE_PROJECT_ID,
  1151. uuid=uuids.instance,
  1152. vcpus=1,
  1153. memory_mb=64,
  1154. metadata={},
  1155. system_metadata={},
  1156. host=self.compute.host,
  1157. vm_state=vm_states.ACTIVE,
  1158. task_state=task_states.DELETING,
  1159. expected_attrs=['metadata', 'system_metadata'])
  1160. bdms = []
  1161. def _create_patch(name, attr):
  1162. patcher = mock.patch.object(name, attr)
  1163. mocked_obj = patcher.start()
  1164. self.addCleanup(patcher.stop)
  1165. return mocked_obj
  1166. mock_delete_instance = _create_patch(self.compute, '_delete_instance')
  1167. mock_set_instance_error_state = _create_patch(
  1168. self.compute, '_set_instance_obj_error_state')
  1169. mock_get_by_instance_uuid.return_value = bdms
  1170. mock_get_by_uuid.return_value = instance
  1171. mock_delete_instance.side_effect = test.TestingException('test')
  1172. self.compute._init_instance(self.context, instance)
  1173. mock_set_instance_error_state.assert_called_once_with(
  1174. self.context, instance)
  1175. def _test_init_instance_reverts_crashed_migrations(self,
  1176. old_vm_state=None):
  1177. power_on = True if (not old_vm_state or
  1178. old_vm_state == vm_states.ACTIVE) else False
  1179. sys_meta = {
  1180. 'old_vm_state': old_vm_state
  1181. }
  1182. instance = fake_instance.fake_instance_obj(
  1183. self.context,
  1184. uuid=uuids.instance,
  1185. vm_state=vm_states.ERROR,
  1186. task_state=task_states.RESIZE_MIGRATING,
  1187. power_state=power_state.SHUTDOWN,
  1188. system_metadata=sys_meta,
  1189. host=self.compute.host,
  1190. expected_attrs=['system_metadata'])
  1191. with test.nested(
  1192. mock.patch.object(objects.Instance, 'get_network_info',
  1193. return_value=network_model.NetworkInfo()),
  1194. mock.patch.object(self.compute.driver, 'plug_vifs'),
  1195. mock.patch.object(self.compute.driver, 'finish_revert_migration'),
  1196. mock.patch.object(self.compute, '_get_instance_block_device_info',
  1197. return_value=[]),
  1198. mock.patch.object(self.compute.driver, 'get_info'),
  1199. mock.patch.object(instance, 'save'),
  1200. mock.patch.object(self.compute, '_retry_reboot',
  1201. return_value=(False, None))
  1202. ) as (mock_get_nw, mock_plug, mock_finish, mock_get_inst,
  1203. mock_get_info, mock_save, mock_retry):
  1204. mock_get_info.side_effect = (
  1205. hardware.InstanceInfo(state=power_state.SHUTDOWN),
  1206. hardware.InstanceInfo(state=power_state.SHUTDOWN))
  1207. self.compute._init_instance(self.context, instance)
  1208. mock_retry.assert_called_once_with(self.context, instance,
  1209. power_state.SHUTDOWN)
  1210. mock_get_nw.assert_called_once_with()
  1211. mock_plug.assert_called_once_with(instance, [])
  1212. mock_get_inst.assert_called_once_with(self.context, instance)
  1213. mock_finish.assert_called_once_with(self.context, instance,
  1214. [], [], power_on)
  1215. mock_save.assert_called_once_with()
  1216. mock_get_info.assert_has_calls(
  1217. [mock.call(instance, use_cache=False),
  1218. mock.call(instance, use_cache=False)])
  1219. self.assertIsNone(instance.task_state)
  1220. def test_init_instance_reverts_crashed_migration_from_active(self):
  1221. self._test_init_instance_reverts_crashed_migrations(
  1222. old_vm_state=vm_states.ACTIVE)
  1223. def test_init_instance_reverts_crashed_migration_from_stopped(self):
  1224. self._test_init_instance_reverts_crashed_migrations(
  1225. old_vm_state=vm_states.STOPPED)
  1226. def test_init_instance_reverts_crashed_migration_no_old_state(self):
  1227. self._test_init_instance_reverts_crashed_migrations(old_vm_state=None)
  1228. def test_init_instance_resets_crashed_live_migration(self):
  1229. instance = fake_instance.fake_instance_obj(
  1230. self.context,
  1231. uuid=uuids.instance,
  1232. vm_state=vm_states.ACTIVE,
  1233. host=self.compute.host,
  1234. task_state=task_states.MIGRATING)
  1235. with test.nested(
  1236. mock.patch.object(instance, 'save'),
  1237. mock.patch('nova.objects.Instance.get_network_info',
  1238. return_value=network_model.NetworkInfo())
  1239. ) as (save, get_nw_info):
  1240. self.compute._init_instance(self.context, instance)
  1241. save.assert_called_once_with(expected_task_state=['migrating'])
  1242. get_nw_info.assert_called_once_with()
  1243. self.assertIsNone(instance.task_state)
  1244. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1245. def _test_init_instance_sets_building_error(self, vm_state,
  1246. task_state=None):
  1247. instance = fake_instance.fake_instance_obj(
  1248. self.context,
  1249. uuid=uuids.instance,
  1250. vm_state=vm_state,
  1251. host=self.compute.host,
  1252. task_state=task_state)
  1253. with mock.patch.object(instance, 'save') as save:
  1254. self.compute._init_instance(self.context, instance)
  1255. save.assert_called_once_with()
  1256. self.assertIsNone(instance.task_state)
  1257. self.assertEqual(vm_states.ERROR, instance.vm_state)
  1258. def test_init_instance_sets_building_error(self):
  1259. self._test_init_instance_sets_building_error(vm_states.BUILDING)
  1260. def test_init_instance_sets_rebuilding_errors(self):
  1261. tasks = [task_states.REBUILDING,
  1262. task_states.REBUILD_BLOCK_DEVICE_MAPPING,
  1263. task_states.REBUILD_SPAWNING]
  1264. vms = [vm_states.ACTIVE, vm_states.STOPPED]
  1265. for vm_state in vms:
  1266. for task_state in tasks:
  1267. self._test_init_instance_sets_building_error(
  1268. vm_state, task_state)
  1269. def _test_init_instance_sets_building_tasks_error(self, instance):
  1270. instance.host = self.compute.host
  1271. with mock.patch.object(instance, 'save') as save:
  1272. self.compute._init_instance(self.context, instance)
  1273. save.assert_called_once_with()
  1274. self.assertIsNone(instance.task_state)
  1275. self.assertEqual(vm_states.ERROR, instance.vm_state)
  1276. def test_init_instance_sets_building_tasks_error_scheduling(self):
  1277. instance = fake_instance.fake_instance_obj(
  1278. self.context,
  1279. uuid=uuids.instance,
  1280. vm_state=None,
  1281. task_state=task_states.SCHEDULING)
  1282. self._test_init_instance_sets_building_tasks_error(instance)
  1283. def test_init_instance_sets_building_tasks_error_block_device(self):
  1284. instance = objects.Instance(self.context)
  1285. instance.uuid = uuids.instance
  1286. instance.vm_state = None
  1287. instance.task_state = task_states.BLOCK_DEVICE_MAPPING
  1288. self._test_init_instance_sets_building_tasks_error(instance)
  1289. def test_init_instance_sets_building_tasks_error_networking(self):
  1290. instance = objects.Instance(self.context)
  1291. instance.uuid = uuids.instance
  1292. instance.vm_state = None
  1293. instance.task_state = task_states.NETWORKING
  1294. self._test_init_instance_sets_building_tasks_error(instance)
  1295. def test_init_instance_sets_building_tasks_error_spawning(self):
  1296. instance = objects.Instance(self.context)
  1297. instance.uuid = uuids.instance
  1298. instance.vm_state = None
  1299. instance.task_state = task_states.SPAWNING
  1300. self._test_init_instance_sets_building_tasks_error(instance)
  1301. def _test_init_instance_cleans_image_states(self, instance):
  1302. with mock.patch.object(instance, 'save') as save:
  1303. self.compute._get_power_state = mock.Mock()
  1304. self.compute.driver.post_interrupted_snapshot_cleanup = mock.Mock()
  1305. instance.info_cache = None
  1306. instance.power_state = power_state.RUNNING
  1307. instance.host = self.compute.host
  1308. self.compute._init_instance(self.context, instance)
  1309. save.assert_called_once_with()
  1310. self.compute.driver.post_interrupted_snapshot_cleanup.\
  1311. assert_called_once_with(self.context, instance)
  1312. self.assertIsNone(instance.task_state)
  1313. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  1314. return_value=power_state.RUNNING)
  1315. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
  1316. def _test_init_instance_cleans_task_states(self, powerstate, state,
  1317. mock_get_uuid, mock_get_power_state):
  1318. instance = objects.Instance(self.context)
  1319. instance.uuid = uuids.instance
  1320. instance.info_cache = None
  1321. instance.power_state = power_state.RUNNING
  1322. instance.vm_state = vm_states.ACTIVE
  1323. instance.task_state = state
  1324. instance.host = self.compute.host
  1325. mock_get_power_state.return_value = powerstate
  1326. self.compute._init_instance(self.context, instance)
  1327. return instance
  1328. def test_init_instance_cleans_image_state_pending_upload(self):
  1329. instance = objects.Instance(self.context)
  1330. instance.uuid = uuids.instance
  1331. instance.vm_state = vm_states.ACTIVE
  1332. instance.task_state = task_states.IMAGE_PENDING_UPLOAD
  1333. self._test_init_instance_cleans_image_states(instance)
  1334. def test_init_instance_cleans_image_state_uploading(self):
  1335. instance = objects.Instance(self.context)
  1336. instance.uuid = uuids.instance
  1337. instance.vm_state = vm_states.ACTIVE
  1338. instance.task_state = task_states.IMAGE_UPLOADING
  1339. self._test_init_instance_cleans_image_states(instance)
  1340. def test_init_instance_cleans_image_state_snapshot(self):
  1341. instance = objects.Instance(self.context)
  1342. instance.uuid = uuids.instance
  1343. instance.vm_state = vm_states.ACTIVE
  1344. instance.task_state = task_states.IMAGE_SNAPSHOT
  1345. self._test_init_instance_cleans_image_states(instance)
  1346. def test_init_instance_cleans_image_state_snapshot_pending(self):
  1347. instance = objects.Instance(self.context)
  1348. instance.uuid = uuids.instance
  1349. instance.vm_state = vm_states.ACTIVE
  1350. instance.task_state = task_states.IMAGE_SNAPSHOT_PENDING
  1351. self._test_init_instance_cleans_image_states(instance)
  1352. @mock.patch.object(objects.Instance, 'save')
  1353. def test_init_instance_cleans_running_pausing(self, mock_save):
  1354. instance = self._test_init_instance_cleans_task_states(
  1355. power_state.RUNNING, task_states.PAUSING)
  1356. mock_save.assert_called_once_with()
  1357. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1358. self.assertIsNone(instance.task_state)
  1359. @mock.patch.object(objects.Instance, 'save')
  1360. def test_init_instance_cleans_running_unpausing(self, mock_save):
  1361. instance = self._test_init_instance_cleans_task_states(
  1362. power_state.RUNNING, task_states.UNPAUSING)
  1363. mock_save.assert_called_once_with()
  1364. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1365. self.assertIsNone(instance.task_state)
  1366. @mock.patch('nova.compute.manager.ComputeManager.unpause_instance')
  1367. def test_init_instance_cleans_paused_unpausing(self, mock_unpause):
  1368. def fake_unpause(context, instance):
  1369. instance.task_state = None
  1370. mock_unpause.side_effect = fake_unpause
  1371. instance = self._test_init_instance_cleans_task_states(
  1372. power_state.PAUSED, task_states.UNPAUSING)
  1373. mock_unpause.assert_called_once_with(self.context, instance)
  1374. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1375. self.assertIsNone(instance.task_state)
  1376. def test_init_instance_deletes_error_deleting_instance(self):
  1377. instance = fake_instance.fake_instance_obj(
  1378. self.context,
  1379. project_id=fakes.FAKE_PROJECT_ID,
  1380. uuid=uuids.instance,
  1381. vcpus=1,
  1382. memory_mb=64,
  1383. vm_state=vm_states.ERROR,
  1384. host=self.compute.host,
  1385. task_state=task_states.DELETING)
  1386. bdms = []
  1387. with test.nested(
  1388. mock.patch.object(objects.BlockDeviceMappingList,
  1389. 'get_by_instance_uuid',
  1390. return_value=bdms),
  1391. mock.patch.object(self.compute, '_delete_instance'),
  1392. mock.patch.object(instance, 'obj_load_attr')
  1393. ) as (mock_get, mock_delete, mock_load):
  1394. self.compute._init_instance(self.context, instance)
  1395. mock_get.assert_called_once_with(self.context, instance.uuid)
  1396. mock_delete.assert_called_once_with(self.context, instance,
  1397. bdms)
  1398. def test_init_instance_resize_prep(self):
  1399. instance = fake_instance.fake_instance_obj(
  1400. self.context,
  1401. uuid=uuids.instance,
  1402. vm_state=vm_states.ACTIVE,
  1403. host=self.compute.host,
  1404. task_state=task_states.RESIZE_PREP,
  1405. power_state=power_state.RUNNING)
  1406. with test.nested(
  1407. mock.patch.object(self.compute, '_get_power_state',
  1408. return_value=power_state.RUNNING),
  1409. mock.patch.object(objects.Instance, 'get_network_info'),
  1410. mock.patch.object(instance, 'save', autospec=True)
  1411. ) as (mock_get_power_state, mock_nw_info, mock_instance_save):
  1412. self.compute._init_instance(self.context, instance)
  1413. mock_instance_save.assert_called_once_with()
  1414. self.assertIsNone(instance.task_state)
  1415. @mock.patch('nova.virt.fake.FakeDriver.power_off')
  1416. @mock.patch.object(compute_utils, 'get_value_from_system_metadata',
  1417. return_value=CONF.shutdown_timeout)
  1418. def test_power_off_values(self, mock_get_metadata, mock_power_off):
  1419. self.flags(shutdown_retry_interval=20, group='compute')
  1420. instance = fake_instance.fake_instance_obj(
  1421. self.context,
  1422. uuid=uuids.instance,
  1423. vm_state=vm_states.ACTIVE,
  1424. task_state=task_states.POWERING_OFF)
  1425. self.compute._power_off_instance(
  1426. self.context, instance,
  1427. clean_shutdown=True)
  1428. mock_power_off.assert_called_once_with(
  1429. instance,
  1430. CONF.shutdown_timeout,
  1431. 20)
  1432. @mock.patch('nova.context.RequestContext.elevated')
  1433. @mock.patch('nova.objects.Instance.get_network_info')
  1434. @mock.patch(
  1435. 'nova.compute.manager.ComputeManager._get_instance_block_device_info')
  1436. @mock.patch('nova.virt.driver.ComputeDriver.destroy')
  1437. @mock.patch('nova.virt.fake.FakeDriver.get_volume_connector')
  1438. @mock.patch('nova.compute.utils.notify_about_instance_action')
  1439. @mock.patch(
  1440. 'nova.compute.manager.ComputeManager._notify_about_instance_usage')
  1441. def test_shutdown_instance_versioned_notifications(self,
  1442. mock_notify_unversioned, mock_notify, mock_connector,
  1443. mock_destroy, mock_blk_device_info, mock_nw_info, mock_elevated):
  1444. mock_elevated.return_value = self.context
  1445. instance = fake_instance.fake_instance_obj(
  1446. self.context,
  1447. uuid=uuids.instance,
  1448. vm_state=vm_states.ERROR,
  1449. task_state=task_states.DELETING)
  1450. bdms = [mock.Mock(id=1, is_volume=True)]
  1451. self.compute._shutdown_instance(self.context, instance, bdms,
  1452. notify=True, try_deallocate_networks=False)
  1453. mock_notify.assert_has_calls([
  1454. mock.call(self.context, instance, 'fake-mini',
  1455. action='shutdown', phase='start', bdms=bdms),
  1456. mock.call(self.context, instance, 'fake-mini',
  1457. action='shutdown', phase='end', bdms=bdms)])
  1458. @mock.patch('nova.context.RequestContext.elevated')
  1459. @mock.patch('nova.objects.Instance.get_network_info')
  1460. @mock.patch(
  1461. 'nova.compute.manager.ComputeManager._get_instance_block_device_info')
  1462. @mock.patch('nova.virt.driver.ComputeDriver.destroy')
  1463. @mock.patch('nova.virt.fake.FakeDriver.get_volume_connector')
  1464. def _test_shutdown_instance_exception(self, exc, mock_connector,
  1465. mock_destroy, mock_blk_device_info, mock_nw_info, mock_elevated):
  1466. mock_connector.side_effect = exc
  1467. mock_elevated.return_value = self.context
  1468. instance = fake_instance.fake_instance_obj(
  1469. self.context,
  1470. uuid=uuids.instance,
  1471. vm_state=vm_states.ERROR,
  1472. task_state=task_states.DELETING)
  1473. bdms = [mock.Mock(id=1, is_volume=True, attachment_id=None)]
  1474. self.compute._shutdown_instance(self.context, instance, bdms,
  1475. notify=False, try_deallocate_networks=False)
  1476. mock_connector.assert_called_once_with(instance)
  1477. def test_shutdown_instance_endpoint_not_found(self):
  1478. exc = cinder_exception.EndpointNotFound
  1479. self._test_shutdown_instance_exception(exc)
  1480. def test_shutdown_instance_client_exception(self):
  1481. exc = cinder_exception.ClientException(code=9001)
  1482. self._test_shutdown_instance_exception(exc)
  1483. def test_shutdown_instance_volume_not_found(self):
  1484. exc = exception.VolumeNotFound(volume_id=42)
  1485. self._test_shutdown_instance_exception(exc)
  1486. def test_shutdown_instance_disk_not_found(self):
  1487. exc = exception.DiskNotFound(location="not\\here")
  1488. self._test_shutdown_instance_exception(exc)
  1489. def test_shutdown_instance_other_exception(self):
  1490. exc = Exception('some other exception')
  1491. self._test_shutdown_instance_exception(exc)
  1492. def _test_init_instance_retries_reboot(self, instance, reboot_type,
  1493. return_power_state):
  1494. instance.host = self.compute.host
  1495. with test.nested(
  1496. mock.patch.object(self.compute, '_get_power_state',
  1497. return_value=return_power_state),
  1498. mock.patch.object(self.compute, 'reboot_instance'),
  1499. mock.patch.object(objects.Instance, 'get_network_info')
  1500. ) as (
  1501. _get_power_state,
  1502. reboot_instance,
  1503. get_network_info
  1504. ):
  1505. self.compute._init_instance(self.context, instance)
  1506. call = mock.call(self.context, instance, block_device_info=None,
  1507. reboot_type=reboot_type)
  1508. reboot_instance.assert_has_calls([call])
  1509. def test_init_instance_retries_reboot_pending(self):
  1510. instance = objects.Instance(self.context)
  1511. instance.uuid = uuids.instance
  1512. instance.task_state = task_states.REBOOT_PENDING
  1513. for state in vm_states.ALLOW_SOFT_REBOOT:
  1514. instance.vm_state = state
  1515. self._test_init_instance_retries_reboot(instance, 'SOFT',
  1516. power_state.RUNNING)
  1517. def test_init_instance_retries_reboot_pending_hard(self):
  1518. instance = objects.Instance(self.context)
  1519. instance.uuid = uuids.instance
  1520. instance.task_state = task_states.REBOOT_PENDING_HARD
  1521. for state in vm_states.ALLOW_HARD_REBOOT:
  1522. # NOTE(dave-mcnally) while a reboot of a vm in error state is
  1523. # possible we don't attempt to recover an error during init
  1524. if state == vm_states.ERROR:
  1525. continue
  1526. instance.vm_state = state
  1527. self._test_init_instance_retries_reboot(instance, 'HARD',
  1528. power_state.RUNNING)
  1529. def test_init_instance_retries_reboot_pending_soft_became_hard(self):
  1530. instance = objects.Instance(self.context)
  1531. instance.uuid = uuids.instance
  1532. instance.task_state = task_states.REBOOT_PENDING
  1533. for state in vm_states.ALLOW_HARD_REBOOT:
  1534. # NOTE(dave-mcnally) while a reboot of a vm in error state is
  1535. # possible we don't attempt to recover an error during init
  1536. if state == vm_states.ERROR:
  1537. continue
  1538. instance.vm_state = state
  1539. with mock.patch.object(instance, 'save'):
  1540. self._test_init_instance_retries_reboot(instance, 'HARD',
  1541. power_state.SHUTDOWN)
  1542. self.assertEqual(task_states.REBOOT_PENDING_HARD,
  1543. instance.task_state)
  1544. def test_init_instance_retries_reboot_started(self):
  1545. instance = objects.Instance(self.context)
  1546. instance.uuid = uuids.instance
  1547. instance.vm_state = vm_states.ACTIVE
  1548. instance.task_state = task_states.REBOOT_STARTED
  1549. with mock.patch.object(instance, 'save'):
  1550. self._test_init_instance_retries_reboot(instance, 'HARD',
  1551. power_state.NOSTATE)
  1552. def test_init_instance_retries_reboot_started_hard(self):
  1553. instance = objects.Instance(self.context)
  1554. instance.uuid = uuids.instance
  1555. instance.vm_state = vm_states.ACTIVE
  1556. instance.task_state = task_states.REBOOT_STARTED_HARD
  1557. self._test_init_instance_retries_reboot(instance, 'HARD',
  1558. power_state.NOSTATE)
  1559. def _test_init_instance_cleans_reboot_state(self, instance):
  1560. instance.host = self.compute.host
  1561. with test.nested(
  1562. mock.patch.object(self.compute, '_get_power_state',
  1563. return_value=power_state.RUNNING),
  1564. mock.patch.object(instance, 'save', autospec=True),
  1565. mock.patch.object(objects.Instance, 'get_network_info')
  1566. ) as (
  1567. _get_power_state,
  1568. instance_save,
  1569. get_network_info
  1570. ):
  1571. self.compute._init_instance(self.context, instance)
  1572. instance_save.assert_called_once_with()
  1573. self.assertIsNone(instance.task_state)
  1574. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1575. def test_init_instance_cleans_image_state_reboot_started(self):
  1576. instance = objects.Instance(self.context)
  1577. instance.uuid = uuids.instance
  1578. instance.vm_state = vm_states.ACTIVE
  1579. instance.task_state = task_states.REBOOT_STARTED
  1580. instance.power_state = power_state.RUNNING
  1581. self._test_init_instance_cleans_reboot_state(instance)
  1582. def test_init_instance_cleans_image_state_reboot_started_hard(self):
  1583. instance = objects.Instance(self.context)
  1584. instance.uuid = uuids.instance
  1585. instance.vm_state = vm_states.ACTIVE
  1586. instance.task_state = task_states.REBOOT_STARTED_HARD
  1587. instance.power_state = power_state.RUNNING
  1588. self._test_init_instance_cleans_reboot_state(instance)
  1589. def test_init_instance_retries_power_off(self):
  1590. instance = objects.Instance(self.context)
  1591. instance.uuid = uuids.instance
  1592. instance.id = 1
  1593. instance.vm_state = vm_states.ACTIVE
  1594. instance.task_state = task_states.POWERING_OFF
  1595. instance.host = self.compute.host
  1596. with mock.patch.object(self.compute, 'stop_instance'):
  1597. self.compute._init_instance(self.context, instance)
  1598. call = mock.call(self.context, instance, True)
  1599. self.compute.stop_instance.assert_has_calls([call])
  1600. def test_init_instance_retries_power_on(self):
  1601. instance = objects.Instance(self.context)
  1602. instance.uuid = uuids.instance
  1603. instance.id = 1
  1604. instance.vm_state = vm_states.ACTIVE
  1605. instance.task_state = task_states.POWERING_ON
  1606. instance.host = self.compute.host
  1607. with mock.patch.object(self.compute, 'start_instance'):
  1608. self.compute._init_instance(self.context, instance)
  1609. call = mock.call(self.context, instance)
  1610. self.compute.start_instance.assert_has_calls([call])
  1611. def test_init_instance_retries_power_on_silent_exception(self):
  1612. instance = objects.Instance(self.context)
  1613. instance.uuid = uuids.instance
  1614. instance.id = 1
  1615. instance.vm_state = vm_states.ACTIVE
  1616. instance.task_state = task_states.POWERING_ON
  1617. instance.host = self.compute.host
  1618. with mock.patch.object(self.compute, 'start_instance',
  1619. return_value=Exception):
  1620. init_return = self.compute._init_instance(self.context, instance)
  1621. call = mock.call(self.context, instance)
  1622. self.compute.start_instance.assert_has_calls([call])
  1623. self.assertIsNone(init_return)
  1624. def test_init_instance_retries_power_off_silent_exception(self):
  1625. instance = objects.Instance(self.context)
  1626. instance.uuid = uuids.instance
  1627. instance.id = 1
  1628. instance.vm_state = vm_states.ACTIVE
  1629. instance.task_state = task_states.POWERING_OFF
  1630. instance.host = self.compute.host
  1631. with mock.patch.object(self.compute, 'stop_instance',
  1632. return_value=Exception):
  1633. init_return = self.compute._init_instance(self.context, instance)
  1634. call = mock.call(self.context, instance, True)
  1635. self.compute.stop_instance.assert_has_calls([call])
  1636. self.assertIsNone(init_return)
  1637. def test_get_power_state(self):
  1638. instance = objects.Instance(self.context)
  1639. instance.uuid = uuids.instance
  1640. instance.id = 1
  1641. instance.vm_state = vm_states.STOPPED
  1642. instance.task_state = None
  1643. instance.host = self.compute.host
  1644. with mock.patch.object(self.compute.driver, 'get_info') as mock_info:
  1645. mock_info.return_value = hardware.InstanceInfo(
  1646. state=power_state.SHUTDOWN)
  1647. res = self.compute._get_power_state(self.context, instance)
  1648. mock_info.assert_called_once_with(instance, use_cache=False)
  1649. self.assertEqual(res, power_state.SHUTDOWN)
  1650. @mock.patch('nova.objects.InstanceList.get_by_filters')
  1651. def test_get_instances_on_driver(self, mock_instance_list):
  1652. driver_instances = []
  1653. for x in range(10):
  1654. driver_instances.append(fake_instance.fake_db_instance())
  1655. def _make_instance_list(db_list):
  1656. return instance_obj._make_instance_list(
  1657. self.context, objects.InstanceList(), db_list, None)
  1658. driver_uuids = [inst['uuid'] for inst in driver_instances]
  1659. mock_instance_list.return_value = _make_instance_list(driver_instances)
  1660. with mock.patch.object(self.compute.driver,
  1661. 'list_instance_uuids') as mock_driver_uuids:
  1662. mock_driver_uuids.return_value = driver_uuids
  1663. result = self.compute._get_instances_on_driver(self.context)
  1664. self.assertEqual([x['uuid'] for x in driver_instances],
  1665. [x['uuid'] for x in result])
  1666. expected_filters = {'uuid': driver_uuids}
  1667. mock_instance_list.assert_called_with(self.context, expected_filters,
  1668. use_slave=True)
  1669. @mock.patch('nova.objects.InstanceList.get_by_filters')
  1670. def test_get_instances_on_driver_empty(self, mock_instance_list):
  1671. with mock.patch.object(self.compute.driver,
  1672. 'list_instance_uuids') as mock_driver_uuids:
  1673. mock_driver_uuids.return_value = []
  1674. result = self.compute._get_instances_on_driver(self.context)
  1675. # Short circuit DB call, get_by_filters should not be called
  1676. self.assertEqual(0, mock_instance_list.call_count)
  1677. self.assertEqual(1, mock_driver_uuids.call_count)
  1678. self.assertEqual([], [x['uuid'] for x in result])
  1679. @mock.patch('nova.objects.InstanceList.get_by_filters')
  1680. def test_get_instances_on_driver_fallback(self, mock_instance_list):
  1681. # Test getting instances when driver doesn't support
  1682. # 'list_instance_uuids'
  1683. self.compute.host = 'host'
  1684. filters = {}
  1685. self.flags(instance_name_template='inst-%i')
  1686. all_instances = []
  1687. driver_instances = []
  1688. for x in range(10):
  1689. instance = fake_instance.fake_db_instance(name='inst-%i' % x,
  1690. id=x)
  1691. if x % 2:
  1692. driver_instances.append(instance)
  1693. all_instances.append(instance)
  1694. def _make_instance_list(db_list):
  1695. return instance_obj._make_instance_list(
  1696. self.context, objects.InstanceList(), db_list, None)
  1697. driver_instance_names = [inst['name'] for inst in driver_instances]
  1698. mock_instance_list.return_value = _make_instance_list(all_instances)
  1699. with test.nested(
  1700. mock.patch.object(self.compute.driver, 'list_instance_uuids'),
  1701. mock.patch.object(self.compute.driver, 'list_instances')
  1702. ) as (
  1703. mock_driver_uuids,
  1704. mock_driver_instances
  1705. ):
  1706. mock_driver_uuids.side_effect = NotImplementedError()
  1707. mock_driver_instances.return_value = driver_instance_names
  1708. result = self.compute._get_instances_on_driver(self.context,
  1709. filters)
  1710. self.assertEqual([x['uuid'] for x in driver_instances],
  1711. [x['uuid'] for x in result])
  1712. expected_filters = {'host': self.compute.host}
  1713. mock_instance_list.assert_called_with(self.context, expected_filters,
  1714. use_slave=True)
  1715. @mock.patch.object(compute_utils, 'notify_usage_exists')
  1716. @mock.patch.object(objects.TaskLog, 'end_task')
  1717. @mock.patch.object(objects.TaskLog, 'begin_task')
  1718. @mock.patch.object(objects.InstanceList, 'get_active_by_window_joined')
  1719. @mock.patch.object(objects.TaskLog, 'get')
  1720. def test_instance_usage_audit(self, mock_get, mock_get_active, mock_begin,
  1721. mock_end, mock_notify):
  1722. instances = [objects.Instance(uuid=uuids.instance)]
  1723. def fake_task_log(*a, **k):
  1724. pass
  1725. def fake_get(*a, **k):
  1726. return instances
  1727. mock_get.side_effect = fake_task_log
  1728. mock_get_active.side_effect = fake_get
  1729. mock_begin.side_effect = fake_task_log
  1730. mock_end.side_effect = fake_task_log
  1731. self.flags(instance_usage_audit=True)
  1732. self.compute._instance_usage_audit(self.context)
  1733. mock_notify.assert_called_once_with(
  1734. self.compute.notifier, self.context, instances[0], 'fake-mini',
  1735. ignore_missing_network_data=False)
  1736. self.assertTrue(mock_get.called)
  1737. self.assertTrue(mock_get_active.called)
  1738. self.assertTrue(mock_begin.called)
  1739. self.assertTrue(mock_end.called)
  1740. @mock.patch.object(objects.InstanceList, 'get_by_host')
  1741. def test_sync_power_states(self, mock_get):
  1742. instance = mock.Mock()
  1743. mock_get.return_value = [instance]
  1744. with mock.patch.object(self.compute._sync_power_pool,
  1745. 'spawn_n') as mock_spawn:
  1746. self.compute._sync_power_states(mock.sentinel.context)
  1747. mock_get.assert_called_with(mock.sentinel.context,
  1748. self.compute.host, expected_attrs=[],
  1749. use_slave=True)
  1750. mock_spawn.assert_called_once_with(mock.ANY, instance)
  1751. @mock.patch('nova.objects.InstanceList.get_by_host', new=mock.Mock())
  1752. @mock.patch('nova.compute.manager.ComputeManager.'
  1753. '_query_driver_power_state_and_sync',
  1754. new_callable=mock.NonCallableMock)
  1755. def test_sync_power_states_virt_driver_not_ready(self, _mock_sync):
  1756. """"Tests that the periodic task exits early if the driver raises
  1757. VirtDriverNotReady.
  1758. """
  1759. with mock.patch.object(
  1760. self.compute.driver, 'get_num_instances',
  1761. side_effect=exception.VirtDriverNotReady) as gni:
  1762. self.compute._sync_power_states(mock.sentinel.context)
  1763. gni.assert_called_once_with()
  1764. def _get_sync_instance(self, power_state, vm_state, task_state=None,
  1765. shutdown_terminate=False):
  1766. instance = objects.Instance()
  1767. instance.uuid = uuids.instance
  1768. instance.power_state = power_state
  1769. instance.vm_state = vm_state
  1770. instance.host = self.compute.host
  1771. instance.task_state = task_state
  1772. instance.shutdown_terminate = shutdown_terminate
  1773. return instance
  1774. @mock.patch.object(objects.Instance, 'refresh')
  1775. def test_sync_instance_power_state_match(self, mock_refresh):
  1776. instance = self._get_sync_instance(power_state.RUNNING,
  1777. vm_states.ACTIVE)
  1778. self.compute._sync_instance_power_state(self.context, instance,
  1779. power_state.RUNNING)
  1780. mock_refresh.assert_called_once_with(use_slave=False)
  1781. @mock.patch.object(fake_driver.FakeDriver, 'get_info')
  1782. @mock.patch.object(objects.Instance, 'refresh')
  1783. @mock.patch.object(objects.Instance, 'save')
  1784. def test_sync_instance_power_state_running_stopped(self, mock_save,
  1785. mock_refresh,
  1786. mock_get_info):
  1787. mock_get_info.return_value = hardware.InstanceInfo(
  1788. state=power_state.SHUTDOWN)
  1789. instance = self._get_sync_instance(power_state.RUNNING,
  1790. vm_states.ACTIVE)
  1791. self.compute._sync_instance_power_state(self.context, instance,
  1792. power_state.SHUTDOWN)
  1793. self.assertEqual(instance.power_state, power_state.SHUTDOWN)
  1794. mock_refresh.assert_called_once_with(use_slave=False)
  1795. self.assertTrue(mock_save.called)
  1796. mock_get_info.assert_called_once_with(instance, use_cache=False)
  1797. def _test_sync_to_stop(self, vm_power_state, vm_state, driver_power_state,
  1798. stop=True, force=False, shutdown_terminate=False):
  1799. instance = self._get_sync_instance(
  1800. vm_power_state, vm_state, shutdown_terminate=shutdown_terminate)
  1801. with test.nested(
  1802. mock.patch.object(objects.Instance, 'refresh'),
  1803. mock.patch.object(objects.Instance, 'save'),
  1804. mock.patch.object(self.compute.compute_api, 'stop'),
  1805. mock.patch.object(self.compute.compute_api, 'delete'),
  1806. mock.patch.object(self.compute.compute_api, 'force_stop'),
  1807. mock.patch.object(self.compute.driver, 'get_info')
  1808. ) as (mock_refresh, mock_save, mock_stop, mock_delete, mock_force,
  1809. mock_get_info):
  1810. mock_get_info.return_value = hardware.InstanceInfo(
  1811. state=driver_power_state)
  1812. self.compute._sync_instance_power_state(self.context, instance,
  1813. driver_power_state)
  1814. if shutdown_terminate:
  1815. mock_delete.assert_called_once_with(self.context, instance)
  1816. elif stop:
  1817. if force:
  1818. mock_force.assert_called_once_with(self.context, instance)
  1819. else:
  1820. mock_stop.assert_called_once_with(self.context, instance)
  1821. if (vm_state == vm_states.ACTIVE and
  1822. vm_power_state in (power_state.SHUTDOWN,
  1823. power_state.CRASHED)):
  1824. mock_get_info.assert_called_once_with(instance,
  1825. use_cache=False)
  1826. mock_refresh.assert_called_once_with(use_slave=False)
  1827. self.assertTrue(mock_save.called)
  1828. def test_sync_instance_power_state_to_stop(self):
  1829. for ps in (power_state.SHUTDOWN, power_state.CRASHED,
  1830. power_state.SUSPENDED):
  1831. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE, ps)
  1832. for ps in (power_state.SHUTDOWN, power_state.CRASHED):
  1833. self._test_sync_to_stop(power_state.PAUSED, vm_states.PAUSED, ps,
  1834. force=True)
  1835. self._test_sync_to_stop(power_state.SHUTDOWN, vm_states.STOPPED,
  1836. power_state.RUNNING, force=True)
  1837. def test_sync_instance_power_state_to_terminate(self):
  1838. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE,
  1839. power_state.SHUTDOWN,
  1840. force=False, shutdown_terminate=True)
  1841. def test_sync_instance_power_state_to_no_stop(self):
  1842. for ps in (power_state.PAUSED, power_state.NOSTATE):
  1843. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE, ps,
  1844. stop=False)
  1845. for vs in (vm_states.SOFT_DELETED, vm_states.DELETED):
  1846. for ps in (power_state.NOSTATE, power_state.SHUTDOWN):
  1847. self._test_sync_to_stop(power_state.RUNNING, vs, ps,
  1848. stop=False)
  1849. @mock.patch('nova.compute.manager.ComputeManager.'
  1850. '_sync_instance_power_state')
  1851. def test_query_driver_power_state_and_sync_pending_task(
  1852. self, mock_sync_power_state):
  1853. with mock.patch.object(self.compute.driver,
  1854. 'get_info') as mock_get_info:
  1855. db_instance = objects.Instance(uuid=uuids.db_instance,
  1856. task_state=task_states.POWERING_OFF)
  1857. self.compute._query_driver_power_state_and_sync(self.context,
  1858. db_instance)
  1859. self.assertFalse(mock_get_info.called)
  1860. self.assertFalse(mock_sync_power_state.called)
  1861. @mock.patch('nova.compute.manager.ComputeManager.'
  1862. '_sync_instance_power_state')
  1863. def test_query_driver_power_state_and_sync_not_found_driver(
  1864. self, mock_sync_power_state):
  1865. error = exception.InstanceNotFound(instance_id=1)
  1866. with mock.patch.object(self.compute.driver,
  1867. 'get_info', side_effect=error) as mock_get_info:
  1868. db_instance = objects.Instance(uuid=uuids.db_instance,
  1869. task_state=None)
  1870. self.compute._query_driver_power_state_and_sync(self.context,
  1871. db_instance)
  1872. mock_get_info.assert_called_once_with(db_instance)
  1873. mock_sync_power_state.assert_called_once_with(self.context,
  1874. db_instance,
  1875. power_state.NOSTATE,
  1876. use_slave=True)
  1877. @mock.patch.object(virt_driver.ComputeDriver, 'delete_instance_files')
  1878. @mock.patch.object(objects.InstanceList, 'get_by_filters')
  1879. def test_run_pending_deletes(self, mock_get, mock_delete):
  1880. self.flags(instance_delete_interval=10)
  1881. class FakeInstance(object):
  1882. def __init__(self, uuid, name, smd):
  1883. self.uuid = uuid
  1884. self.name = name
  1885. self.system_metadata = smd
  1886. self.cleaned = False
  1887. def __getitem__(self, name):
  1888. return getattr(self, name)
  1889. def save(self):
  1890. pass
  1891. def _fake_get(ctx, filter, expected_attrs, use_slave):
  1892. mock_get.assert_called_once_with(
  1893. {'read_deleted': 'yes'},
  1894. {'deleted': True, 'soft_deleted': False, 'host': 'fake-mini',
  1895. 'cleaned': False},
  1896. expected_attrs=['system_metadata'],
  1897. use_slave=True)
  1898. return [a, b, c]
  1899. a = FakeInstance(uuids.instanceA, 'apple', {'clean_attempts': '100'})
  1900. b = FakeInstance(uuids.instanceB, 'orange', {'clean_attempts': '3'})
  1901. c = FakeInstance(uuids.instanceC, 'banana', {})
  1902. mock_get.side_effect = _fake_get
  1903. mock_delete.side_effect = [True, False]
  1904. self.compute._run_pending_deletes({})
  1905. self.assertFalse(a.cleaned)
  1906. self.assertEqual('100', a.system_metadata['clean_attempts'])
  1907. self.assertTrue(b.cleaned)
  1908. self.assertEqual('4', b.system_metadata['clean_attempts'])
  1909. self.assertFalse(c.cleaned)
  1910. self.assertEqual('1', c.system_metadata['clean_attempts'])
  1911. mock_delete.assert_has_calls([mock.call(mock.ANY),
  1912. mock.call(mock.ANY)])
  1913. @mock.patch.object(objects.Migration, 'obj_as_admin')
  1914. @mock.patch.object(objects.Migration, 'save')
  1915. @mock.patch.object(objects.MigrationList, 'get_by_filters')
  1916. @mock.patch.object(objects.InstanceList, 'get_by_filters')
  1917. def _test_cleanup_incomplete_migrations(self, inst_host,
  1918. mock_inst_get_by_filters,
  1919. mock_migration_get_by_filters,
  1920. mock_save, mock_obj_as_admin):
  1921. def fake_inst(context, uuid, host):
  1922. inst = objects.Instance(context)
  1923. inst.uuid = uuid
  1924. inst.host = host
  1925. return inst
  1926. def fake_migration(uuid, status, inst_uuid, src_host, dest_host):
  1927. migration = objects.Migration()
  1928. migration.uuid = uuid
  1929. migration.status = status
  1930. migration.instance_uuid = inst_uuid
  1931. migration.source_compute = src_host
  1932. migration.dest_compute = dest_host
  1933. return migration
  1934. fake_instances = [fake_inst(self.context, uuids.instance_1, inst_host),
  1935. fake_inst(self.context, uuids.instance_2, inst_host)]
  1936. fake_migrations = [fake_migration(uuids.mig1, 'error',
  1937. uuids.instance_1,
  1938. 'fake-host', 'fake-mini'),
  1939. fake_migration(uuids.mig2, 'error',
  1940. uuids.instance_2,
  1941. 'fake-host', 'fake-mini')]
  1942. mock_migration_get_by_filters.return_value = fake_migrations
  1943. mock_inst_get_by_filters.return_value = fake_instances
  1944. with mock.patch.object(self.compute.driver, 'delete_instance_files'):
  1945. self.compute._cleanup_incomplete_migrations(self.context)
  1946. # Ensure that migration status is set to 'failed' after instance
  1947. # files deletion for those instances whose instance.host is not
  1948. # same as compute host where periodic task is running.
  1949. for inst in fake_instances:
  1950. if inst.host != CONF.host:
  1951. for mig in fake_migrations:
  1952. if inst.uuid == mig.instance_uuid:
  1953. self.assertEqual('failed', mig.status)
  1954. def test_cleanup_incomplete_migrations_dest_node(self):
  1955. """Test to ensure instance files are deleted from destination node.
  1956. If instance gets deleted during resizing/revert-resizing operation,
  1957. in that case instance files gets deleted from instance.host (source
  1958. host here), but there is possibility that instance files could be
  1959. present on destination node.
  1960. This test ensures that `_cleanup_incomplete_migration` periodic
  1961. task deletes orphaned instance files from destination compute node.
  1962. """
  1963. self.flags(host='fake-mini')
  1964. self._test_cleanup_incomplete_migrations('fake-host')
  1965. def test_cleanup_incomplete_migrations_source_node(self):
  1966. """Test to ensure instance files are deleted from source node.
  1967. If instance gets deleted during resizing/revert-resizing operation,
  1968. in that case instance files gets deleted from instance.host (dest
  1969. host here), but there is possibility that instance files could be
  1970. present on source node.
  1971. This test ensures that `_cleanup_incomplete_migration` periodic
  1972. task deletes orphaned instance files from source compute node.
  1973. """
  1974. self.flags(host='fake-host')
  1975. self._test_cleanup_incomplete_migrations('fake-mini')
  1976. def test_attach_interface_failure(self):
  1977. # Test that the fault methods are invoked when an attach fails
  1978. db_instance = fake_instance.fake_db_instance()
  1979. f_instance = objects.Instance._from_db_object(self.context,
  1980. objects.Instance(),
  1981. db_instance)
  1982. e = exception.InterfaceAttachFailed(instance_uuid=f_instance.uuid)
  1983. @mock.patch.object(compute_utils, 'EventReporter')
  1984. @mock.patch.object(compute_utils, 'notify_about_instance_action')
  1985. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  1986. @mock.patch.object(self.compute.network_api,
  1987. 'allocate_port_for_instance',
  1988. side_effect=e)
  1989. @mock.patch.object(self.compute, '_instance_update',
  1990. side_effect=lambda *a, **k: {})
  1991. def do_test(update, meth, add_fault, notify, event):
  1992. self.assertRaises(exception.InterfaceAttachFailed,
  1993. self.compute.attach_interface,
  1994. self.context, f_instance, 'net_id', 'port_id',
  1995. None, None)
  1996. add_fault.assert_has_calls([
  1997. mock.call(self.context, f_instance, e,
  1998. mock.ANY)])
  1999. event.assert_called_once_with(
  2000. self.context, 'compute_attach_interface', CONF.host,
  2001. f_instance.uuid)
  2002. with mock.patch.dict(self.compute.driver.capabilities,
  2003. supports_attach_interface=True):
  2004. do_test()
  2005. def test_detach_interface_failure(self):
  2006. # Test that the fault methods are invoked when a detach fails
  2007. # Build test data that will cause a PortNotFound exception
  2008. nw_info = network_model.NetworkInfo([])
  2009. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2010. instance_uuid=uuids.instance)
  2011. f_instance = objects.Instance(id=3, uuid=uuids.instance,
  2012. info_cache=info_cache)
  2013. @mock.patch.object(compute_utils, 'EventReporter')
  2014. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  2015. @mock.patch.object(self.compute, '_set_instance_obj_error_state')
  2016. def do_test(meth, add_fault, event):
  2017. self.assertRaises(exception.PortNotFound,
  2018. self.compute.detach_interface,
  2019. self.context, f_instance, 'port_id')
  2020. add_fault.assert_has_calls(
  2021. [mock.call(self.context, f_instance, mock.ANY, mock.ANY)])
  2022. event.assert_called_once_with(
  2023. self.context, 'compute_detach_interface', CONF.host,
  2024. f_instance.uuid)
  2025. do_test()
  2026. @mock.patch('nova.compute.manager.LOG.log')
  2027. @mock.patch.object(compute_utils, 'EventReporter')
  2028. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  2029. @mock.patch.object(compute_utils, 'notify_about_instance_action')
  2030. def test_detach_interface_instance_not_found(self, mock_notify, mock_fault,
  2031. mock_event, mock_log):
  2032. nw_info = network_model.NetworkInfo([
  2033. network_model.VIF(uuids.port_id)])
  2034. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2035. instance_uuid=uuids.instance)
  2036. instance = objects.Instance(id=1, uuid=uuids.instance,
  2037. info_cache=info_cache)
  2038. with mock.patch.object(self.compute.driver, 'detach_interface',
  2039. side_effect=exception.InstanceNotFound(
  2040. instance_id=uuids.instance)):
  2041. self.assertRaises(exception.InterfaceDetachFailed,
  2042. self.compute.detach_interface,
  2043. self.context, instance, uuids.port_id)
  2044. self.assertEqual(1, mock_log.call_count)
  2045. self.assertEqual(logging.DEBUG, mock_log.call_args[0][0])
  2046. @mock.patch.object(compute_utils, 'EventReporter')
  2047. @mock.patch.object(virt_driver.ComputeDriver, 'get_volume_connector',
  2048. return_value={})
  2049. @mock.patch.object(manager.ComputeManager, '_instance_update',
  2050. return_value={})
  2051. @mock.patch.object(db, 'instance_fault_create')
  2052. @mock.patch.object(db, 'block_device_mapping_update')
  2053. @mock.patch.object(db,
  2054. 'block_device_mapping_get_by_instance_and_volume_id')
  2055. @mock.patch.object(cinder.API, 'migrate_volume_completion')
  2056. @mock.patch.object(cinder.API, 'terminate_connection')
  2057. @mock.patch.object(cinder.API, 'unreserve_volume')
  2058. @mock.patch.object(cinder.API, 'get')
  2059. @mock.patch.object(cinder.API, 'roll_detaching')
  2060. @mock.patch.object(compute_utils, 'notify_about_volume_swap')
  2061. def _test_swap_volume(self, mock_notify, mock_roll_detaching,
  2062. mock_cinder_get, mock_unreserve_volume,
  2063. mock_terminate_connection,
  2064. mock_migrate_volume_completion,
  2065. mock_bdm_get, mock_bdm_update,
  2066. mock_instance_fault_create,
  2067. mock_instance_update,
  2068. mock_get_volume_connector,
  2069. mock_event,
  2070. expected_exception=None):
  2071. # This test ensures that volume_id arguments are passed to volume_api
  2072. # and that volume states are OK
  2073. volumes = {}
  2074. volumes[uuids.old_volume] = {'id': uuids.old_volume,
  2075. 'display_name': 'old_volume',
  2076. 'status': 'detaching',
  2077. 'size': 1}
  2078. volumes[uuids.new_volume] = {'id': uuids.new_volume,
  2079. 'display_name': 'new_volume',
  2080. 'status': 'available',
  2081. 'size': 2}
  2082. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  2083. {'device_name': '/dev/vdb', 'source_type': 'volume',
  2084. 'destination_type': 'volume',
  2085. 'instance_uuid': uuids.instance,
  2086. 'connection_info': '{"foo": "bar"}',
  2087. 'volume_id': uuids.old_volume,
  2088. 'attachment_id': None})
  2089. def fake_vol_api_roll_detaching(context, volume_id):
  2090. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  2091. if volumes[volume_id]['status'] == 'detaching':
  2092. volumes[volume_id]['status'] = 'in-use'
  2093. def fake_vol_api_func(context, volume, *args):
  2094. self.assertTrue(uuidutils.is_uuid_like(volume))
  2095. return {}
  2096. def fake_vol_get(context, volume_id):
  2097. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  2098. return volumes[volume_id]
  2099. def fake_vol_unreserve(context, volume_id):
  2100. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  2101. if volumes[volume_id]['status'] == 'attaching':
  2102. volumes[volume_id]['status'] = 'available'
  2103. def fake_vol_migrate_volume_completion(context, old_volume_id,
  2104. new_volume_id, error=False):
  2105. self.assertTrue(uuidutils.is_uuid_like(old_volume_id))
  2106. self.assertTrue(uuidutils.is_uuid_like(new_volume_id))
  2107. volumes[old_volume_id]['status'] = 'in-use'
  2108. return {'save_volume_id': new_volume_id}
  2109. def fake_block_device_mapping_update(ctxt, id, updates, legacy):
  2110. self.assertEqual(2, updates['volume_size'])
  2111. return fake_bdm
  2112. mock_roll_detaching.side_effect = fake_vol_api_roll_detaching
  2113. mock_terminate_connection.side_effect = fake_vol_api_func
  2114. mock_cinder_get.side_effect = fake_vol_get
  2115. mock_migrate_volume_completion.side_effect = (
  2116. fake_vol_migrate_volume_completion)
  2117. mock_unreserve_volume.side_effect = fake_vol_unreserve
  2118. mock_bdm_get.return_value = fake_bdm
  2119. mock_bdm_update.side_effect = fake_block_device_mapping_update
  2120. mock_instance_fault_create.return_value = (
  2121. test_instance_fault.fake_faults['fake-uuid'][0])
  2122. instance1 = fake_instance.fake_instance_obj(
  2123. self.context, **{'uuid': uuids.instance})
  2124. if expected_exception:
  2125. volumes[uuids.old_volume]['status'] = 'detaching'
  2126. volumes[uuids.new_volume]['status'] = 'attaching'
  2127. self.assertRaises(expected_exception, self.compute.swap_volume,
  2128. self.context, uuids.old_volume, uuids.new_volume,
  2129. instance1, None)
  2130. self.assertEqual('in-use', volumes[uuids.old_volume]['status'])
  2131. self.assertEqual('available', volumes[uuids.new_volume]['status'])
  2132. self.assertEqual(2, mock_notify.call_count)
  2133. mock_notify.assert_any_call(
  2134. test.MatchType(context.RequestContext), instance1,
  2135. self.compute.host,
  2136. fields.NotificationPhase.START,
  2137. uuids.old_volume, uuids.new_volume)
  2138. mock_notify.assert_any_call(
  2139. test.MatchType(context.RequestContext), instance1,
  2140. self.compute.host,
  2141. fields.NotificationPhase.ERROR,
  2142. uuids.old_volume, uuids.new_volume,
  2143. test.MatchType(expected_exception), mock.ANY)
  2144. else:
  2145. self.compute.swap_volume(self.context, uuids.old_volume,
  2146. uuids.new_volume, instance1, None)
  2147. self.assertEqual(volumes[uuids.old_volume]['status'], 'in-use')
  2148. self.assertEqual(2, mock_notify.call_count)
  2149. mock_notify.assert_any_call(test.MatchType(context.RequestContext),
  2150. instance1, self.compute.host,
  2151. fields.NotificationPhase.START,
  2152. uuids.old_volume, uuids.new_volume)
  2153. mock_notify.assert_any_call(test.MatchType(context.RequestContext),
  2154. instance1, self.compute.host,
  2155. fields.NotificationPhase.END,
  2156. uuids.old_volume, uuids.new_volume)
  2157. mock_event.assert_called_once_with(self.context,
  2158. 'compute_swap_volume',
  2159. CONF.host,
  2160. instance1.uuid)
  2161. def _assert_volume_api(self, context, volume, *args):
  2162. self.assertTrue(uuidutils.is_uuid_like(volume))
  2163. return {}
  2164. def _assert_swap_volume(self, context, old_connection_info,
  2165. new_connection_info, instance, mountpoint,
  2166. resize_to):
  2167. self.assertEqual(2, resize_to)
  2168. @mock.patch.object(cinder.API, 'initialize_connection')
  2169. @mock.patch.object(fake_driver.FakeDriver, 'swap_volume')
  2170. def test_swap_volume_volume_api_usage(self, mock_swap_volume,
  2171. mock_initialize_connection):
  2172. mock_initialize_connection.side_effect = self._assert_volume_api
  2173. mock_swap_volume.side_effect = self._assert_swap_volume
  2174. self._test_swap_volume()
  2175. @mock.patch.object(cinder.API, 'initialize_connection')
  2176. @mock.patch.object(fake_driver.FakeDriver, 'swap_volume',
  2177. side_effect=test.TestingException())
  2178. def test_swap_volume_with_compute_driver_exception(
  2179. self, mock_swap_volume, mock_initialize_connection):
  2180. mock_initialize_connection.side_effect = self._assert_volume_api
  2181. self._test_swap_volume(expected_exception=test.TestingException)
  2182. @mock.patch.object(cinder.API, 'initialize_connection',
  2183. side_effect=test.TestingException())
  2184. @mock.patch.object(fake_driver.FakeDriver, 'swap_volume')
  2185. def test_swap_volume_with_initialize_connection_exception(
  2186. self, mock_swap_volume, mock_initialize_connection):
  2187. self._test_swap_volume(expected_exception=test.TestingException)
  2188. @mock.patch('nova.compute.utils.notify_about_volume_swap')
  2189. @mock.patch(
  2190. 'nova.db.api.block_device_mapping_get_by_instance_and_volume_id')
  2191. @mock.patch('nova.db.api.block_device_mapping_update')
  2192. @mock.patch('nova.volume.cinder.API.get')
  2193. @mock.patch('nova.virt.libvirt.LibvirtDriver.get_volume_connector')
  2194. @mock.patch('nova.compute.manager.ComputeManager._swap_volume')
  2195. def test_swap_volume_delete_on_termination_flag(self, swap_volume_mock,
  2196. volume_connector_mock,
  2197. get_volume_mock,
  2198. update_bdm_mock,
  2199. get_bdm_mock,
  2200. notify_mock):
  2201. # This test ensures that delete_on_termination flag arguments
  2202. # are reserved
  2203. volumes = {}
  2204. old_volume_id = uuids.fake
  2205. volumes[old_volume_id] = {'id': old_volume_id,
  2206. 'display_name': 'old_volume',
  2207. 'status': 'detaching',
  2208. 'size': 2}
  2209. new_volume_id = uuids.fake_2
  2210. volumes[new_volume_id] = {'id': new_volume_id,
  2211. 'display_name': 'new_volume',
  2212. 'status': 'available',
  2213. 'size': 2}
  2214. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  2215. {'device_name': '/dev/vdb', 'source_type': 'volume',
  2216. 'destination_type': 'volume',
  2217. 'instance_uuid': uuids.instance,
  2218. 'delete_on_termination': True,
  2219. 'connection_info': '{"foo": "bar"}',
  2220. 'attachment_id': None})
  2221. comp_ret = {'save_volume_id': old_volume_id}
  2222. new_info = {"foo": "bar", "serial": old_volume_id}
  2223. swap_volume_mock.return_value = (comp_ret, new_info)
  2224. volume_connector_mock.return_value = {}
  2225. update_bdm_mock.return_value = fake_bdm
  2226. get_bdm_mock.return_value = fake_bdm
  2227. get_volume_mock.return_value = volumes[old_volume_id]
  2228. self.compute.swap_volume(self.context, old_volume_id, new_volume_id,
  2229. fake_instance.fake_instance_obj(self.context,
  2230. **{'uuid': uuids.instance}),
  2231. None)
  2232. update_values = {'no_device': False,
  2233. 'connection_info': jsonutils.dumps(new_info),
  2234. 'volume_id': old_volume_id,
  2235. 'source_type': u'volume',
  2236. 'snapshot_id': None,
  2237. 'destination_type': u'volume'}
  2238. update_bdm_mock.assert_called_once_with(mock.ANY, mock.ANY,
  2239. update_values, legacy=False)
  2240. @mock.patch.object(compute_utils, 'notify_about_volume_swap')
  2241. @mock.patch.object(objects.BlockDeviceMapping,
  2242. 'get_by_volume_and_instance')
  2243. @mock.patch('nova.volume.cinder.API.get')
  2244. @mock.patch('nova.volume.cinder.API.attachment_update')
  2245. @mock.patch('nova.volume.cinder.API.attachment_delete')
  2246. @mock.patch('nova.volume.cinder.API.attachment_complete')
  2247. @mock.patch('nova.volume.cinder.API.migrate_volume_completion',
  2248. return_value={'save_volume_id': uuids.old_volume_id})
  2249. def test_swap_volume_with_new_attachment_id_cinder_migrate_true(
  2250. self, migrate_volume_completion, attachment_complete,
  2251. attachment_delete, attachment_update, get_volume, get_bdm,
  2252. notify_about_volume_swap):
  2253. """Tests a swap volume operation with a new style volume attachment
  2254. passed in from the compute API, and the case that Cinder initiated
  2255. the swap volume because of a volume retype situation. This is a happy
  2256. path test. Since it is a retype there is no volume size change.
  2257. """
  2258. bdm = objects.BlockDeviceMapping(
  2259. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2260. attachment_id=uuids.old_attachment_id,
  2261. connection_info='{"data": {}}', volume_size=1)
  2262. old_volume = {
  2263. 'id': uuids.old_volume_id, 'size': 1, 'status': 'retyping',
  2264. 'migration_status': 'migrating', 'multiattach': False
  2265. }
  2266. new_volume = {
  2267. 'id': uuids.new_volume_id, 'size': 1, 'status': 'reserved',
  2268. 'migration_status': 'migrating', 'multiattach': False
  2269. }
  2270. attachment_update.return_value = {"connection_info": {"data": {}}}
  2271. get_bdm.return_value = bdm
  2272. get_volume.side_effect = (old_volume, new_volume)
  2273. instance = fake_instance.fake_instance_obj(self.context)
  2274. with test.nested(
  2275. mock.patch.object(self.context, 'elevated',
  2276. return_value=self.context),
  2277. mock.patch.object(self.compute.driver, 'get_volume_connector',
  2278. return_value=mock.sentinel.connector),
  2279. mock.patch.object(bdm, 'save')
  2280. ) as (
  2281. mock_elevated, mock_get_volume_connector, mock_save
  2282. ):
  2283. self.compute.swap_volume(
  2284. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2285. instance, uuids.new_attachment_id)
  2286. # Assert the expected calls.
  2287. get_bdm.assert_called_once_with(
  2288. self.context, uuids.old_volume_id, instance.uuid)
  2289. # We updated the new attachment with the host connector.
  2290. attachment_update.assert_called_once_with(
  2291. self.context, uuids.new_attachment_id, mock.sentinel.connector,
  2292. bdm.device_name)
  2293. # We tell Cinder that the new volume is connected
  2294. attachment_complete.assert_called_once_with(
  2295. self.context, uuids.new_attachment_id)
  2296. # After a successful swap volume, we deleted the old attachment.
  2297. attachment_delete.assert_called_once_with(
  2298. self.context, uuids.old_attachment_id)
  2299. # After a successful swap volume, we tell Cinder so it can complete
  2300. # the retype operation.
  2301. migrate_volume_completion.assert_called_once_with(
  2302. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2303. error=False)
  2304. # The BDM should have been updated. Since it's a retype, the old
  2305. # volume ID is returned from Cinder so that's what goes into the
  2306. # BDM but the new attachment ID is saved.
  2307. mock_save.assert_called_once_with()
  2308. self.assertEqual(uuids.old_volume_id, bdm.volume_id)
  2309. self.assertEqual(uuids.new_attachment_id, bdm.attachment_id)
  2310. self.assertEqual(1, bdm.volume_size)
  2311. self.assertEqual(uuids.old_volume_id,
  2312. jsonutils.loads(bdm.connection_info)['serial'])
  2313. @mock.patch.object(compute_utils, 'notify_about_volume_swap')
  2314. @mock.patch.object(objects.BlockDeviceMapping,
  2315. 'get_by_volume_and_instance')
  2316. @mock.patch('nova.volume.cinder.API.get')
  2317. @mock.patch('nova.volume.cinder.API.attachment_update')
  2318. @mock.patch('nova.volume.cinder.API.attachment_delete')
  2319. @mock.patch('nova.volume.cinder.API.attachment_complete')
  2320. @mock.patch('nova.volume.cinder.API.migrate_volume_completion')
  2321. def test_swap_volume_with_new_attachment_id_cinder_migrate_false(
  2322. self, migrate_volume_completion, attachment_complete,
  2323. attachment_delete, attachment_update, get_volume, get_bdm,
  2324. notify_about_volume_swap):
  2325. """Tests a swap volume operation with a new style volume attachment
  2326. passed in from the compute API, and the case that Cinder did not
  2327. initiate the swap volume. This is a happy path test. Since it is not a
  2328. retype we also change the size.
  2329. """
  2330. bdm = objects.BlockDeviceMapping(
  2331. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2332. attachment_id=uuids.old_attachment_id,
  2333. connection_info='{"data": {}}')
  2334. old_volume = {
  2335. 'id': uuids.old_volume_id, 'size': 1, 'status': 'detaching',
  2336. 'multiattach': False
  2337. }
  2338. new_volume = {
  2339. 'id': uuids.new_volume_id, 'size': 2, 'status': 'reserved',
  2340. 'multiattach': False
  2341. }
  2342. attachment_update.return_value = {"connection_info": {"data": {}}}
  2343. get_bdm.return_value = bdm
  2344. get_volume.side_effect = (old_volume, new_volume)
  2345. instance = fake_instance.fake_instance_obj(self.context)
  2346. with test.nested(
  2347. mock.patch.object(self.context, 'elevated',
  2348. return_value=self.context),
  2349. mock.patch.object(self.compute.driver, 'get_volume_connector',
  2350. return_value=mock.sentinel.connector),
  2351. mock.patch.object(bdm, 'save')
  2352. ) as (
  2353. mock_elevated, mock_get_volume_connector, mock_save
  2354. ):
  2355. self.compute.swap_volume(
  2356. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2357. instance, uuids.new_attachment_id)
  2358. # Assert the expected calls.
  2359. get_bdm.assert_called_once_with(
  2360. self.context, uuids.old_volume_id, instance.uuid)
  2361. # We updated the new attachment with the host connector.
  2362. attachment_update.assert_called_once_with(
  2363. self.context, uuids.new_attachment_id, mock.sentinel.connector,
  2364. bdm.device_name)
  2365. # We tell Cinder that the new volume is connected
  2366. attachment_complete.assert_called_once_with(
  2367. self.context, uuids.new_attachment_id)
  2368. # After a successful swap volume, we deleted the old attachment.
  2369. attachment_delete.assert_called_once_with(
  2370. self.context, uuids.old_attachment_id)
  2371. # After a successful swap volume, since it was not a
  2372. # Cinder-initiated call, we don't call migrate_volume_completion.
  2373. migrate_volume_completion.assert_not_called()
  2374. # The BDM should have been updated. Since it's a not a retype, the
  2375. # volume_id is now the new volume ID.
  2376. mock_save.assert_called_once_with()
  2377. self.assertEqual(uuids.new_volume_id, bdm.volume_id)
  2378. self.assertEqual(uuids.new_attachment_id, bdm.attachment_id)
  2379. self.assertEqual(2, bdm.volume_size)
  2380. new_conn_info = jsonutils.loads(bdm.connection_info)
  2381. self.assertEqual(uuids.new_volume_id, new_conn_info['serial'])
  2382. self.assertNotIn('multiattach', new_conn_info)
  2383. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  2384. @mock.patch.object(compute_utils, 'notify_about_volume_swap')
  2385. @mock.patch.object(objects.BlockDeviceMapping,
  2386. 'get_by_volume_and_instance')
  2387. @mock.patch('nova.volume.cinder.API.get')
  2388. @mock.patch('nova.volume.cinder.API.attachment_update',
  2389. side_effect=exception.VolumeAttachmentNotFound(
  2390. attachment_id=uuids.new_attachment_id))
  2391. @mock.patch('nova.volume.cinder.API.roll_detaching')
  2392. @mock.patch('nova.volume.cinder.API.attachment_delete')
  2393. @mock.patch('nova.volume.cinder.API.migrate_volume_completion')
  2394. def test_swap_volume_with_new_attachment_id_attachment_update_fails(
  2395. self, migrate_volume_completion, attachment_delete, roll_detaching,
  2396. attachment_update, get_volume, get_bdm, notify_about_volume_swap,
  2397. add_instance_fault_from_exc):
  2398. """Tests a swap volume operation with a new style volume attachment
  2399. passed in from the compute API, and the case that Cinder initiated
  2400. the swap volume because of a volume migrate situation. This is a
  2401. negative test where attachment_update fails.
  2402. """
  2403. bdm = objects.BlockDeviceMapping(
  2404. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2405. attachment_id=uuids.old_attachment_id,
  2406. connection_info='{"data": {}}')
  2407. old_volume = {
  2408. 'id': uuids.old_volume_id, 'size': 1, 'status': 'in-use',
  2409. 'migration_status': 'migrating', 'multiattach': False
  2410. }
  2411. new_volume = {
  2412. 'id': uuids.new_volume_id, 'size': 1, 'status': 'reserved',
  2413. 'migration_status': 'migrating', 'multiattach': False
  2414. }
  2415. get_bdm.return_value = bdm
  2416. get_volume.side_effect = (old_volume, new_volume)
  2417. instance = fake_instance.fake_instance_obj(self.context)
  2418. with test.nested(
  2419. mock.patch.object(self.context, 'elevated',
  2420. return_value=self.context),
  2421. mock.patch.object(self.compute.driver, 'get_volume_connector',
  2422. return_value=mock.sentinel.connector)
  2423. ) as (
  2424. mock_elevated, mock_get_volume_connector
  2425. ):
  2426. self.assertRaises(
  2427. exception.VolumeAttachmentNotFound, self.compute.swap_volume,
  2428. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2429. instance, uuids.new_attachment_id)
  2430. # Assert the expected calls.
  2431. get_bdm.assert_called_once_with(
  2432. self.context, uuids.old_volume_id, instance.uuid)
  2433. # We tried to update the new attachment with the host connector.
  2434. attachment_update.assert_called_once_with(
  2435. self.context, uuids.new_attachment_id, mock.sentinel.connector,
  2436. bdm.device_name)
  2437. # After a failure, we rollback the detaching status of the old
  2438. # volume.
  2439. roll_detaching.assert_called_once_with(
  2440. self.context, uuids.old_volume_id)
  2441. # After a failure, we deleted the new attachment.
  2442. attachment_delete.assert_called_once_with(
  2443. self.context, uuids.new_attachment_id)
  2444. # After a failure for a Cinder-initiated swap volume, we called
  2445. # migrate_volume_completion to let Cinder know things blew up.
  2446. migrate_volume_completion.assert_called_once_with(
  2447. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2448. error=True)
  2449. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  2450. @mock.patch.object(compute_utils, 'notify_about_volume_swap')
  2451. @mock.patch.object(objects.BlockDeviceMapping,
  2452. 'get_by_volume_and_instance')
  2453. @mock.patch('nova.volume.cinder.API.get')
  2454. @mock.patch('nova.volume.cinder.API.attachment_update')
  2455. @mock.patch('nova.volume.cinder.API.roll_detaching')
  2456. @mock.patch('nova.volume.cinder.API.attachment_delete')
  2457. @mock.patch('nova.volume.cinder.API.migrate_volume_completion')
  2458. def test_swap_volume_with_new_attachment_id_driver_swap_fails(
  2459. self, migrate_volume_completion, attachment_delete, roll_detaching,
  2460. attachment_update, get_volume, get_bdm, notify_about_volume_swap,
  2461. add_instance_fault_from_exc):
  2462. """Tests a swap volume operation with a new style volume attachment
  2463. passed in from the compute API, and the case that Cinder did not
  2464. initiate the swap volume. This is a negative test where the compute
  2465. driver swap_volume method fails.
  2466. """
  2467. bdm = objects.BlockDeviceMapping(
  2468. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2469. attachment_id=uuids.old_attachment_id,
  2470. connection_info='{"data": {}}')
  2471. old_volume = {
  2472. 'id': uuids.old_volume_id, 'size': 1, 'status': 'detaching',
  2473. 'multiattach': False
  2474. }
  2475. new_volume = {
  2476. 'id': uuids.new_volume_id, 'size': 2, 'status': 'reserved',
  2477. 'multiattach': False
  2478. }
  2479. attachment_update.return_value = {"connection_info": {"data": {}}}
  2480. get_bdm.return_value = bdm
  2481. get_volume.side_effect = (old_volume, new_volume)
  2482. instance = fake_instance.fake_instance_obj(self.context)
  2483. with test.nested(
  2484. mock.patch.object(self.context, 'elevated',
  2485. return_value=self.context),
  2486. mock.patch.object(self.compute.driver, 'get_volume_connector',
  2487. return_value=mock.sentinel.connector),
  2488. mock.patch.object(self.compute.driver, 'swap_volume',
  2489. side_effect=test.TestingException('yikes'))
  2490. ) as (
  2491. mock_elevated, mock_get_volume_connector, mock_driver_swap
  2492. ):
  2493. self.assertRaises(
  2494. test.TestingException, self.compute.swap_volume,
  2495. self.context, uuids.old_volume_id, uuids.new_volume_id,
  2496. instance, uuids.new_attachment_id)
  2497. # Assert the expected calls.
  2498. # The new connection_info has the new_volume_id as the serial.
  2499. new_cinfo = mock_driver_swap.call_args[0][2]
  2500. self.assertIn('serial', new_cinfo)
  2501. self.assertEqual(uuids.new_volume_id, new_cinfo['serial'])
  2502. get_bdm.assert_called_once_with(
  2503. self.context, uuids.old_volume_id, instance.uuid)
  2504. # We updated the new attachment with the host connector.
  2505. attachment_update.assert_called_once_with(
  2506. self.context, uuids.new_attachment_id, mock.sentinel.connector,
  2507. bdm.device_name)
  2508. # After a failure, we rollback the detaching status of the old
  2509. # volume.
  2510. roll_detaching.assert_called_once_with(
  2511. self.context, uuids.old_volume_id)
  2512. # After a failed swap volume, we deleted the new attachment.
  2513. attachment_delete.assert_called_once_with(
  2514. self.context, uuids.new_attachment_id)
  2515. # After a failed swap volume, since it was not a
  2516. # Cinder-initiated call, we don't call migrate_volume_completion.
  2517. migrate_volume_completion.assert_not_called()
  2518. @mock.patch('nova.volume.cinder.API.attachment_update')
  2519. def test_swap_volume_with_multiattach(self, attachment_update):
  2520. """Tests swap volume where the volume being swapped-to supports
  2521. multiattach as well as the compute driver, so the attachment for the
  2522. new volume (created in the API) is updated with the host connector
  2523. and the new_connection_info is updated with the multiattach flag.
  2524. """
  2525. bdm = objects.BlockDeviceMapping(
  2526. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2527. attachment_id=uuids.old_attachment_id,
  2528. connection_info='{"data": {}}')
  2529. new_volume = {
  2530. 'id': uuids.new_volume_id, 'size': 2, 'status': 'reserved',
  2531. 'multiattach': True
  2532. }
  2533. attachment_update.return_value = {"connection_info": {"data": {}}}
  2534. connector = mock.sentinel.connector
  2535. with mock.patch.dict(self.compute.driver.capabilities,
  2536. {'supports_multiattach': True}):
  2537. _, new_cinfo = self.compute._init_volume_connection(
  2538. self.context, new_volume, uuids.old_volume_id,
  2539. connector, bdm, uuids.new_attachment_id, bdm.device_name)
  2540. self.assertEqual(uuids.new_volume_id, new_cinfo['serial'])
  2541. self.assertIn('multiattach', new_cinfo)
  2542. self.assertTrue(new_cinfo['multiattach'])
  2543. attachment_update.assert_called_once_with(
  2544. self.context, uuids.new_attachment_id, connector,
  2545. bdm.device_name)
  2546. def test_swap_volume_with_multiattach_no_driver_support(self):
  2547. """Tests a swap volume scenario where the new volume being swapped-to
  2548. supports multiattach but the virt driver does not, so swap volume
  2549. fails.
  2550. """
  2551. bdm = objects.BlockDeviceMapping(
  2552. volume_id=uuids.old_volume_id, device_name='/dev/vda',
  2553. attachment_id=uuids.old_attachment_id,
  2554. connection_info='{"data": {}}')
  2555. new_volume = {
  2556. 'id': uuids.new_volume_id, 'size': 2, 'status': 'reserved',
  2557. 'multiattach': True
  2558. }
  2559. connector = {'host': 'localhost'}
  2560. with mock.patch.dict(self.compute.driver.capabilities,
  2561. {'supports_multiattach': False}):
  2562. self.assertRaises(exception.MultiattachNotSupportedByVirtDriver,
  2563. self.compute._init_volume_connection,
  2564. self.context, new_volume, uuids.old_volume_id,
  2565. connector, bdm, uuids.new_attachment_id,
  2566. bdm.device_name)
  2567. @mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
  2568. @mock.patch.object(fake_driver.FakeDriver,
  2569. 'check_can_live_migrate_source')
  2570. @mock.patch.object(manager.ComputeManager,
  2571. '_get_instance_block_device_info')
  2572. @mock.patch.object(compute_utils, 'is_volume_backed_instance')
  2573. @mock.patch.object(compute_utils, 'EventReporter')
  2574. def test_check_can_live_migrate_source(self, mock_event, mock_volume,
  2575. mock_get_inst, mock_check,
  2576. mock_get_bdms):
  2577. fake_bdms = objects.BlockDeviceMappingList()
  2578. mock_get_bdms.return_value = fake_bdms
  2579. is_volume_backed = 'volume_backed'
  2580. dest_check_data = migrate_data_obj.LiveMigrateData()
  2581. db_instance = fake_instance.fake_db_instance()
  2582. instance = objects.Instance._from_db_object(
  2583. self.context, objects.Instance(), db_instance)
  2584. mock_volume.return_value = is_volume_backed
  2585. mock_get_inst.return_value = {'block_device_mapping': 'fake'}
  2586. self.compute.check_can_live_migrate_source(
  2587. self.context, instance=instance,
  2588. dest_check_data=dest_check_data)
  2589. mock_event.assert_called_once_with(
  2590. self.context, 'compute_check_can_live_migrate_source', CONF.host,
  2591. instance.uuid)
  2592. mock_check.assert_called_once_with(self.context, instance,
  2593. dest_check_data,
  2594. {'block_device_mapping': 'fake'})
  2595. mock_volume.assert_called_once_with(self.context, instance, fake_bdms)
  2596. mock_get_inst.assert_called_once_with(self.context, instance,
  2597. refresh_conn_info=False,
  2598. bdms=fake_bdms)
  2599. self.assertTrue(dest_check_data.is_volume_backed)
  2600. def _test_check_can_live_migrate_destination(self, do_raise=False):
  2601. db_instance = fake_instance.fake_db_instance(host='fake-host')
  2602. instance = objects.Instance._from_db_object(
  2603. self.context, objects.Instance(), db_instance)
  2604. instance.host = 'fake-host'
  2605. block_migration = 'block_migration'
  2606. disk_over_commit = 'disk_over_commit'
  2607. src_info = 'src_info'
  2608. dest_info = 'dest_info'
  2609. dest_check_data = dict(foo='bar')
  2610. mig_data = dict(cow='moo')
  2611. with test.nested(
  2612. mock.patch.object(self.compute, '_get_compute_info'),
  2613. mock.patch.object(self.compute.driver,
  2614. 'check_can_live_migrate_destination'),
  2615. mock.patch.object(self.compute.compute_rpcapi,
  2616. 'check_can_live_migrate_source'),
  2617. mock.patch.object(self.compute.driver,
  2618. 'cleanup_live_migration_destination_check'),
  2619. mock.patch.object(db, 'instance_fault_create'),
  2620. mock.patch.object(compute_utils, 'EventReporter')
  2621. ) as (mock_get, mock_check_dest, mock_check_src, mock_check_clean,
  2622. mock_fault_create, mock_event):
  2623. mock_get.side_effect = (src_info, dest_info)
  2624. mock_check_dest.return_value = dest_check_data
  2625. if do_raise:
  2626. mock_check_src.side_effect = test.TestingException
  2627. mock_fault_create.return_value = \
  2628. test_instance_fault.fake_faults['fake-uuid'][0]
  2629. else:
  2630. mock_check_src.return_value = mig_data
  2631. result = self.compute.check_can_live_migrate_destination(
  2632. self.context, instance=instance,
  2633. block_migration=block_migration,
  2634. disk_over_commit=disk_over_commit)
  2635. if do_raise:
  2636. mock_fault_create.assert_called_once_with(self.context,
  2637. mock.ANY)
  2638. mock_check_src.assert_called_once_with(self.context, instance,
  2639. dest_check_data)
  2640. mock_check_clean.assert_called_once_with(self.context,
  2641. dest_check_data)
  2642. mock_get.assert_has_calls([mock.call(self.context, 'fake-host'),
  2643. mock.call(self.context, CONF.host)])
  2644. mock_check_dest.assert_called_once_with(self.context, instance,
  2645. src_info, dest_info, block_migration, disk_over_commit)
  2646. self.assertEqual(mig_data, result)
  2647. mock_event.assert_called_once_with(
  2648. self.context, 'compute_check_can_live_migrate_destination',
  2649. CONF.host, instance.uuid)
  2650. def test_check_can_live_migrate_destination_success(self):
  2651. self._test_check_can_live_migrate_destination()
  2652. def test_check_can_live_migrate_destination_fail(self):
  2653. self.assertRaises(
  2654. test.TestingException,
  2655. self._test_check_can_live_migrate_destination,
  2656. do_raise=True)
  2657. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  2658. def test_prepare_for_instance_event(self, lock_name_mock):
  2659. inst_obj = objects.Instance(uuid=uuids.instance)
  2660. result = self.compute.instance_events.prepare_for_instance_event(
  2661. inst_obj, 'test-event', None)
  2662. self.assertIn(uuids.instance, self.compute.instance_events._events)
  2663. self.assertIn(('test-event', None),
  2664. self.compute.instance_events._events[uuids.instance])
  2665. self.assertEqual(
  2666. result,
  2667. self.compute.instance_events._events[uuids.instance]
  2668. [('test-event', None)])
  2669. self.assertTrue(hasattr(result, 'send'))
  2670. lock_name_mock.assert_called_once_with(inst_obj)
  2671. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  2672. def test_pop_instance_event(self, lock_name_mock):
  2673. event = eventlet_event.Event()
  2674. self.compute.instance_events._events = {
  2675. uuids.instance: {
  2676. ('network-vif-plugged', None): event,
  2677. }
  2678. }
  2679. inst_obj = objects.Instance(uuid=uuids.instance)
  2680. event_obj = objects.InstanceExternalEvent(name='network-vif-plugged',
  2681. tag=None)
  2682. result = self.compute.instance_events.pop_instance_event(inst_obj,
  2683. event_obj)
  2684. self.assertEqual(result, event)
  2685. lock_name_mock.assert_called_once_with(inst_obj)
  2686. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  2687. def test_clear_events_for_instance(self, lock_name_mock):
  2688. event = eventlet_event.Event()
  2689. self.compute.instance_events._events = {
  2690. uuids.instance: {
  2691. ('test-event', None): event,
  2692. }
  2693. }
  2694. inst_obj = objects.Instance(uuid=uuids.instance)
  2695. result = self.compute.instance_events.clear_events_for_instance(
  2696. inst_obj)
  2697. self.assertEqual(result, {'test-event-None': event})
  2698. lock_name_mock.assert_called_once_with(inst_obj)
  2699. def test_instance_events_lock_name(self):
  2700. inst_obj = objects.Instance(uuid=uuids.instance)
  2701. result = self.compute.instance_events._lock_name(inst_obj)
  2702. self.assertEqual(result, "%s-events" % uuids.instance)
  2703. def test_prepare_for_instance_event_again(self):
  2704. inst_obj = objects.Instance(uuid=uuids.instance)
  2705. self.compute.instance_events.prepare_for_instance_event(
  2706. inst_obj, 'test-event', None)
  2707. # A second attempt will avoid creating a new list; make sure we
  2708. # get the current list
  2709. result = self.compute.instance_events.prepare_for_instance_event(
  2710. inst_obj, 'test-event', None)
  2711. self.assertIn(uuids.instance, self.compute.instance_events._events)
  2712. self.assertIn(('test-event', None),
  2713. self.compute.instance_events._events[uuids.instance])
  2714. self.assertEqual(
  2715. result,
  2716. self.compute.instance_events._events[uuids.instance]
  2717. [('test-event', None)])
  2718. self.assertTrue(hasattr(result, 'send'))
  2719. def test_process_instance_event(self):
  2720. event = eventlet_event.Event()
  2721. self.compute.instance_events._events = {
  2722. uuids.instance: {
  2723. ('network-vif-plugged', None): event,
  2724. }
  2725. }
  2726. inst_obj = objects.Instance(uuid=uuids.instance)
  2727. event_obj = objects.InstanceExternalEvent(name='network-vif-plugged',
  2728. tag=None)
  2729. self.compute._process_instance_event(inst_obj, event_obj)
  2730. self.assertTrue(event.ready())
  2731. self.assertEqual(event_obj, event.wait())
  2732. self.assertEqual({}, self.compute.instance_events._events)
  2733. @ddt.data(task_states.DELETING,
  2734. task_states.MIGRATING)
  2735. @mock.patch('nova.compute.manager.LOG')
  2736. def test_process_instance_event_expected_task(self, task_state, mock_log):
  2737. """Tests that we don't log a warning when we get a
  2738. network-vif-unplugged event for an instance that's undergoing a task
  2739. state transition that will generate the expected event.
  2740. """
  2741. inst_obj = objects.Instance(uuid=uuids.instance,
  2742. vm_state=vm_states.ACTIVE,
  2743. task_state=task_state)
  2744. event_obj = objects.InstanceExternalEvent(name='network-vif-unplugged',
  2745. tag=uuids.port_id)
  2746. with mock.patch.object(self.compute.instance_events,
  2747. 'pop_instance_event', return_value=None):
  2748. self.compute._process_instance_event(inst_obj, event_obj)
  2749. # assert we logged at debug level
  2750. mock_log.debug.assert_called()
  2751. self.assertThat(mock_log.debug.call_args[0][0],
  2752. testtools.matchers.MatchesRegex(
  2753. 'Received event .* for instance with task_state '
  2754. '%s'))
  2755. @mock.patch('nova.compute.manager.LOG')
  2756. def test_process_instance_event_unexpected_warning(self, mock_log):
  2757. """Tests that we log a warning when we get an unexpected event."""
  2758. inst_obj = objects.Instance(uuid=uuids.instance,
  2759. vm_state=vm_states.ACTIVE,
  2760. task_state=None)
  2761. event_obj = objects.InstanceExternalEvent(name='network-vif-unplugged',
  2762. tag=uuids.port_id)
  2763. with mock.patch.object(self.compute.instance_events,
  2764. 'pop_instance_event', return_value=None):
  2765. self.compute._process_instance_event(inst_obj, event_obj)
  2766. # assert we logged at warning level
  2767. mock_log.warning.assert_called()
  2768. self.assertThat(mock_log.warning.call_args[0][0],
  2769. testtools.matchers.MatchesRegex(
  2770. 'Received unexpected event .* for '
  2771. 'instance with vm_state .* and '
  2772. 'task_state .*.'))
  2773. def test_process_instance_vif_deleted_event(self):
  2774. vif1 = fake_network_cache_model.new_vif()
  2775. vif1['id'] = '1'
  2776. vif2 = fake_network_cache_model.new_vif()
  2777. vif2['id'] = '2'
  2778. nw_info = network_model.NetworkInfo([vif1, vif2])
  2779. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2780. instance_uuid=uuids.instance)
  2781. inst_obj = objects.Instance(id=3, uuid=uuids.instance,
  2782. info_cache=info_cache)
  2783. @mock.patch.object(manager.base_net_api,
  2784. 'update_instance_cache_with_nw_info')
  2785. @mock.patch.object(self.compute.driver, 'detach_interface')
  2786. def do_test(detach_interface, update_instance_cache_with_nw_info):
  2787. self.compute._process_instance_vif_deleted_event(self.context,
  2788. inst_obj,
  2789. vif2['id'])
  2790. update_instance_cache_with_nw_info.assert_called_once_with(
  2791. self.compute.network_api,
  2792. self.context,
  2793. inst_obj,
  2794. nw_info=[vif1])
  2795. detach_interface.assert_called_once_with(self.context,
  2796. inst_obj, vif2)
  2797. do_test()
  2798. def test_process_instance_vif_deleted_event_not_implemented_error(self):
  2799. """Tests the case where driver.detach_interface raises
  2800. NotImplementedError.
  2801. """
  2802. vif = fake_network_cache_model.new_vif()
  2803. nw_info = network_model.NetworkInfo([vif])
  2804. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2805. instance_uuid=uuids.instance)
  2806. inst_obj = objects.Instance(id=3, uuid=uuids.instance,
  2807. info_cache=info_cache)
  2808. @mock.patch.object(manager.base_net_api,
  2809. 'update_instance_cache_with_nw_info')
  2810. @mock.patch.object(self.compute.driver, 'detach_interface',
  2811. side_effect=NotImplementedError)
  2812. def do_test(detach_interface, update_instance_cache_with_nw_info):
  2813. self.compute._process_instance_vif_deleted_event(
  2814. self.context, inst_obj, vif['id'])
  2815. update_instance_cache_with_nw_info.assert_called_once_with(
  2816. self.compute.network_api, self.context, inst_obj, nw_info=[])
  2817. detach_interface.assert_called_once_with(
  2818. self.context, inst_obj, vif)
  2819. do_test()
  2820. @mock.patch('nova.compute.manager.LOG.info') # This is needed for py35.
  2821. @mock.patch('nova.compute.manager.LOG.log')
  2822. def test_process_instance_vif_deleted_event_instance_not_found(
  2823. self, mock_log, mock_log_info):
  2824. """Tests the case where driver.detach_interface raises
  2825. InstanceNotFound.
  2826. """
  2827. vif = fake_network_cache_model.new_vif()
  2828. nw_info = network_model.NetworkInfo([vif])
  2829. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2830. instance_uuid=uuids.instance)
  2831. inst_obj = objects.Instance(id=3, uuid=uuids.instance,
  2832. info_cache=info_cache)
  2833. @mock.patch.object(manager.base_net_api,
  2834. 'update_instance_cache_with_nw_info')
  2835. @mock.patch.object(self.compute.driver, 'detach_interface',
  2836. side_effect=exception.InstanceNotFound(
  2837. instance_id=uuids.instance))
  2838. def do_test(detach_interface, update_instance_cache_with_nw_info):
  2839. self.compute._process_instance_vif_deleted_event(
  2840. self.context, inst_obj, vif['id'])
  2841. update_instance_cache_with_nw_info.assert_called_once_with(
  2842. self.compute.network_api, self.context, inst_obj, nw_info=[])
  2843. detach_interface.assert_called_once_with(
  2844. self.context, inst_obj, vif)
  2845. # LOG.log should have been called with a DEBUG level message.
  2846. self.assertEqual(1, mock_log.call_count, mock_log.mock_calls)
  2847. self.assertEqual(logging.DEBUG, mock_log.call_args[0][0])
  2848. do_test()
  2849. def test_extend_volume(self):
  2850. inst_obj = objects.Instance(id=3, uuid=uuids.instance)
  2851. connection_info = {'foo': 'bar'}
  2852. bdm = objects.BlockDeviceMapping(
  2853. source_type='volume',
  2854. destination_type='volume',
  2855. volume_id=uuids.volume_id,
  2856. volume_size=10,
  2857. instance_uuid=uuids.instance,
  2858. device_name='/dev/vda',
  2859. connection_info=jsonutils.dumps(connection_info))
  2860. @mock.patch.object(self.compute, 'volume_api')
  2861. @mock.patch.object(self.compute.driver, 'extend_volume')
  2862. @mock.patch.object(objects.BlockDeviceMapping,
  2863. 'get_by_volume_and_instance')
  2864. @mock.patch.object(objects.BlockDeviceMapping, 'save')
  2865. def do_test(bdm_save, bdm_get_by_vol_and_inst, extend_volume,
  2866. volume_api):
  2867. bdm_get_by_vol_and_inst.return_value = bdm
  2868. volume_api.get.return_value = {'size': 20}
  2869. self.compute.extend_volume(
  2870. self.context, inst_obj, uuids.volume_id)
  2871. bdm_save.assert_called_once_with()
  2872. extend_volume.assert_called_once_with(
  2873. connection_info, inst_obj)
  2874. do_test()
  2875. def test_extend_volume_not_implemented_error(self):
  2876. """Tests the case where driver.extend_volume raises
  2877. NotImplementedError.
  2878. """
  2879. inst_obj = objects.Instance(id=3, uuid=uuids.instance)
  2880. connection_info = {'foo': 'bar'}
  2881. bdm = objects.BlockDeviceMapping(
  2882. source_type='volume',
  2883. destination_type='volume',
  2884. volume_id=uuids.volume_id,
  2885. volume_size=10,
  2886. instance_uuid=uuids.instance,
  2887. device_name='/dev/vda',
  2888. connection_info=jsonutils.dumps(connection_info))
  2889. @mock.patch.object(self.compute, 'volume_api')
  2890. @mock.patch.object(objects.BlockDeviceMapping,
  2891. 'get_by_volume_and_instance')
  2892. @mock.patch.object(objects.BlockDeviceMapping, 'save')
  2893. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  2894. def do_test(add_fault_mock, bdm_save, bdm_get_by_vol_and_inst,
  2895. volume_api):
  2896. bdm_get_by_vol_and_inst.return_value = bdm
  2897. volume_api.get.return_value = {'size': 20}
  2898. self.assertRaises(
  2899. exception.ExtendVolumeNotSupported,
  2900. self.compute.extend_volume,
  2901. self.context, inst_obj, uuids.volume_id)
  2902. add_fault_mock.assert_called_once_with(
  2903. self.context, inst_obj, mock.ANY, mock.ANY)
  2904. with mock.patch.dict(self.compute.driver.capabilities,
  2905. supports_extend_volume=False):
  2906. do_test()
  2907. def test_extend_volume_volume_not_found(self):
  2908. """Tests the case where driver.extend_volume tries to extend
  2909. a volume not attached to the specified instance.
  2910. """
  2911. inst_obj = objects.Instance(id=3, uuid=uuids.instance)
  2912. @mock.patch.object(objects.BlockDeviceMapping,
  2913. 'get_by_volume_and_instance',
  2914. side_effect=exception.NotFound())
  2915. def do_test(bdm_get_by_vol_and_inst):
  2916. self.compute.extend_volume(
  2917. self.context, inst_obj, uuids.volume_id)
  2918. do_test()
  2919. def test_external_instance_event(self):
  2920. instances = [
  2921. objects.Instance(id=1, uuid=uuids.instance_1),
  2922. objects.Instance(id=2, uuid=uuids.instance_2),
  2923. objects.Instance(id=3, uuid=uuids.instance_3),
  2924. objects.Instance(id=4, uuid=uuids.instance_4)]
  2925. events = [
  2926. objects.InstanceExternalEvent(name='network-changed',
  2927. tag='tag1',
  2928. instance_uuid=uuids.instance_1),
  2929. objects.InstanceExternalEvent(name='network-vif-plugged',
  2930. instance_uuid=uuids.instance_2,
  2931. tag='tag2'),
  2932. objects.InstanceExternalEvent(name='network-vif-deleted',
  2933. instance_uuid=uuids.instance_3,
  2934. tag='tag3'),
  2935. objects.InstanceExternalEvent(name='volume-extended',
  2936. instance_uuid=uuids.instance_4,
  2937. tag='tag4')]
  2938. @mock.patch.object(self.compute,
  2939. 'extend_volume')
  2940. @mock.patch.object(self.compute, '_process_instance_vif_deleted_event')
  2941. @mock.patch.object(self.compute.network_api, 'get_instance_nw_info')
  2942. @mock.patch.object(self.compute, '_process_instance_event')
  2943. def do_test(_process_instance_event, get_instance_nw_info,
  2944. _process_instance_vif_deleted_event, extend_volume):
  2945. self.compute.external_instance_event(self.context,
  2946. instances, events)
  2947. get_instance_nw_info.assert_called_once_with(self.context,
  2948. instances[0],
  2949. refresh_vif_id='tag1')
  2950. _process_instance_event.assert_called_once_with(instances[1],
  2951. events[1])
  2952. _process_instance_vif_deleted_event.assert_called_once_with(
  2953. self.context, instances[2], events[2].tag)
  2954. extend_volume.assert_called_once_with(
  2955. self.context, instances[3], events[3].tag)
  2956. do_test()
  2957. def test_external_instance_event_with_exception(self):
  2958. vif1 = fake_network_cache_model.new_vif()
  2959. vif1['id'] = '1'
  2960. vif2 = fake_network_cache_model.new_vif()
  2961. vif2['id'] = '2'
  2962. nw_info = network_model.NetworkInfo([vif1, vif2])
  2963. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  2964. instance_uuid=uuids.instance_2)
  2965. instances = [
  2966. objects.Instance(id=1, uuid=uuids.instance_1),
  2967. objects.Instance(id=2, uuid=uuids.instance_2,
  2968. info_cache=info_cache),
  2969. objects.Instance(id=3, uuid=uuids.instance_3),
  2970. # instance_4 doesn't have info_cache set so it will be lazy-loaded
  2971. # and blow up with an InstanceNotFound error.
  2972. objects.Instance(id=4, uuid=uuids.instance_4),
  2973. objects.Instance(id=5, uuid=uuids.instance_5),
  2974. ]
  2975. events = [
  2976. objects.InstanceExternalEvent(name='network-changed',
  2977. tag='tag1',
  2978. instance_uuid=uuids.instance_1),
  2979. objects.InstanceExternalEvent(name='network-vif-deleted',
  2980. instance_uuid=uuids.instance_2,
  2981. tag='2'),
  2982. objects.InstanceExternalEvent(name='network-vif-plugged',
  2983. instance_uuid=uuids.instance_3,
  2984. tag='tag3'),
  2985. objects.InstanceExternalEvent(name='network-vif-deleted',
  2986. instance_uuid=uuids.instance_4,
  2987. tag='tag4'),
  2988. objects.InstanceExternalEvent(name='volume-extended',
  2989. instance_uuid=uuids.instance_5,
  2990. tag='tag5'),
  2991. ]
  2992. # Make sure all the four events are handled despite the exceptions in
  2993. # processing events 1, 2, 4 and 5.
  2994. @mock.patch.object(objects.BlockDeviceMapping,
  2995. 'get_by_volume_and_instance',
  2996. side_effect=exception.InstanceNotFound(
  2997. instance_id=uuids.instance_5))
  2998. @mock.patch.object(instances[3], 'obj_load_attr',
  2999. side_effect=exception.InstanceNotFound(
  3000. instance_id=uuids.instance_4))
  3001. @mock.patch.object(manager.base_net_api,
  3002. 'update_instance_cache_with_nw_info')
  3003. @mock.patch.object(self.compute.driver, 'detach_interface',
  3004. side_effect=exception.NovaException)
  3005. @mock.patch.object(self.compute.network_api, 'get_instance_nw_info',
  3006. side_effect=exception.InstanceInfoCacheNotFound(
  3007. instance_uuid=uuids.instance_1))
  3008. @mock.patch.object(self.compute, '_process_instance_event')
  3009. def do_test(_process_instance_event, get_instance_nw_info,
  3010. detach_interface, update_instance_cache_with_nw_info,
  3011. obj_load_attr, bdm_get_by_vol_and_inst):
  3012. self.compute.external_instance_event(self.context,
  3013. instances, events)
  3014. get_instance_nw_info.assert_called_once_with(self.context,
  3015. instances[0],
  3016. refresh_vif_id='tag1')
  3017. update_instance_cache_with_nw_info.assert_called_once_with(
  3018. self.compute.network_api,
  3019. self.context,
  3020. instances[1],
  3021. nw_info=[vif1])
  3022. detach_interface.assert_called_once_with(self.context,
  3023. instances[1], vif2)
  3024. _process_instance_event.assert_called_once_with(instances[2],
  3025. events[2])
  3026. obj_load_attr.assert_called_once_with('info_cache')
  3027. bdm_get_by_vol_and_inst.assert_called_once_with(
  3028. self.context, 'tag5', instances[4].uuid)
  3029. do_test()
  3030. def test_cancel_all_events(self):
  3031. inst = objects.Instance(uuid=uuids.instance)
  3032. fake_eventlet_event = mock.MagicMock()
  3033. self.compute.instance_events._events = {
  3034. inst.uuid: {
  3035. ('network-vif-plugged', uuids.portid): fake_eventlet_event,
  3036. }
  3037. }
  3038. self.compute.instance_events.cancel_all_events()
  3039. # call it again to make sure we handle that gracefully
  3040. self.compute.instance_events.cancel_all_events()
  3041. self.assertTrue(fake_eventlet_event.send.called)
  3042. event = fake_eventlet_event.send.call_args_list[0][0][0]
  3043. self.assertEqual('network-vif-plugged', event.name)
  3044. self.assertEqual(uuids.portid, event.tag)
  3045. self.assertEqual('failed', event.status)
  3046. def test_cleanup_cancels_all_events(self):
  3047. with mock.patch.object(self.compute, 'instance_events') as mock_ev:
  3048. self.compute.cleanup_host()
  3049. mock_ev.cancel_all_events.assert_called_once_with()
  3050. def test_cleanup_blocks_new_events(self):
  3051. instance = objects.Instance(uuid=uuids.instance)
  3052. self.compute.instance_events.cancel_all_events()
  3053. callback = mock.MagicMock()
  3054. body = mock.MagicMock()
  3055. with self.compute.virtapi.wait_for_instance_event(
  3056. instance, [('network-vif-plugged', 'bar')],
  3057. error_callback=callback):
  3058. body()
  3059. self.assertTrue(body.called)
  3060. callback.assert_called_once_with('network-vif-plugged-bar', instance)
  3061. def test_pop_events_fails_gracefully(self):
  3062. inst = objects.Instance(uuid=uuids.instance)
  3063. event = mock.MagicMock()
  3064. self.compute.instance_events._events = None
  3065. self.assertIsNone(
  3066. self.compute.instance_events.pop_instance_event(inst, event))
  3067. def test_clear_events_fails_gracefully(self):
  3068. inst = objects.Instance(uuid=uuids.instance)
  3069. self.compute.instance_events._events = None
  3070. self.assertEqual(
  3071. self.compute.instance_events.clear_events_for_instance(inst), {})
  3072. def test_retry_reboot_pending_soft(self):
  3073. instance = objects.Instance(self.context)
  3074. instance.uuid = uuids.instance
  3075. instance.task_state = task_states.REBOOT_PENDING
  3076. instance.vm_state = vm_states.ACTIVE
  3077. allow_reboot, reboot_type = self.compute._retry_reboot(
  3078. context, instance, power_state.RUNNING)
  3079. self.assertTrue(allow_reboot)
  3080. self.assertEqual(reboot_type, 'SOFT')
  3081. def test_retry_reboot_pending_hard(self):
  3082. instance = objects.Instance(self.context)
  3083. instance.uuid = uuids.instance
  3084. instance.task_state = task_states.REBOOT_PENDING_HARD
  3085. instance.vm_state = vm_states.ACTIVE
  3086. allow_reboot, reboot_type = self.compute._retry_reboot(
  3087. context, instance, power_state.RUNNING)
  3088. self.assertTrue(allow_reboot)
  3089. self.assertEqual(reboot_type, 'HARD')
  3090. def test_retry_reboot_starting_soft_off(self):
  3091. instance = objects.Instance(self.context)
  3092. instance.uuid = uuids.instance
  3093. instance.task_state = task_states.REBOOT_STARTED
  3094. allow_reboot, reboot_type = self.compute._retry_reboot(
  3095. context, instance, power_state.NOSTATE)
  3096. self.assertTrue(allow_reboot)
  3097. self.assertEqual(reboot_type, 'HARD')
  3098. def test_retry_reboot_starting_hard_off(self):
  3099. instance = objects.Instance(self.context)
  3100. instance.uuid = uuids.instance
  3101. instance.task_state = task_states.REBOOT_STARTED_HARD
  3102. allow_reboot, reboot_type = self.compute._retry_reboot(
  3103. context, instance, power_state.NOSTATE)
  3104. self.assertTrue(allow_reboot)
  3105. self.assertEqual(reboot_type, 'HARD')
  3106. def test_retry_reboot_starting_hard_on(self):
  3107. instance = objects.Instance(self.context)
  3108. instance.uuid = uuids.instance
  3109. instance.task_state = task_states.REBOOT_STARTED_HARD
  3110. allow_reboot, reboot_type = self.compute._retry_reboot(
  3111. context, instance, power_state.RUNNING)
  3112. self.assertFalse(allow_reboot)
  3113. self.assertEqual(reboot_type, 'HARD')
  3114. def test_retry_reboot_no_reboot(self):
  3115. instance = objects.Instance(self.context)
  3116. instance.uuid = uuids.instance
  3117. instance.task_state = 'bar'
  3118. allow_reboot, reboot_type = self.compute._retry_reboot(
  3119. context, instance, power_state.RUNNING)
  3120. self.assertFalse(allow_reboot)
  3121. self.assertEqual(reboot_type, 'HARD')
  3122. @mock.patch('nova.objects.BlockDeviceMapping.get_by_volume_and_instance')
  3123. def test_remove_volume_connection(self, bdm_get):
  3124. inst = mock.Mock()
  3125. inst.uuid = uuids.instance_uuid
  3126. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  3127. {'source_type': 'volume', 'destination_type': 'volume',
  3128. 'volume_id': uuids.volume_id, 'device_name': '/dev/vdb',
  3129. 'connection_info': '{"test": "test"}'})
  3130. bdm = objects.BlockDeviceMapping(context=self.context, **fake_bdm)
  3131. bdm_get.return_value = bdm
  3132. with test.nested(
  3133. mock.patch.object(self.compute, 'volume_api'),
  3134. mock.patch.object(self.compute, 'driver'),
  3135. mock.patch.object(driver_bdm_volume, 'driver_detach'),
  3136. ) as (mock_volume_api, mock_virt_driver, mock_driver_detach):
  3137. connector = mock.Mock()
  3138. def fake_driver_detach(context, instance, volume_api, virt_driver):
  3139. # This is just here to validate the function signature.
  3140. pass
  3141. # There should be an easier way to do this with autospec...
  3142. mock_driver_detach.side_effect = fake_driver_detach
  3143. mock_virt_driver.get_volume_connector.return_value = connector
  3144. self.compute.remove_volume_connection(self.context,
  3145. uuids.volume_id, inst)
  3146. bdm_get.assert_called_once_with(self.context, uuids.volume_id,
  3147. uuids.instance_uuid)
  3148. mock_driver_detach.assert_called_once_with(self.context, inst,
  3149. mock_volume_api,
  3150. mock_virt_driver)
  3151. mock_volume_api.terminate_connection.assert_called_once_with(
  3152. self.context, uuids.volume_id, connector)
  3153. def test_remove_volume_connection_cinder_v3_api(self):
  3154. instance = fake_instance.fake_instance_obj(self.context,
  3155. uuid=uuids.instance)
  3156. volume_id = uuids.volume
  3157. vol_bdm = fake_block_device.fake_bdm_object(
  3158. self.context,
  3159. {'source_type': 'volume', 'destination_type': 'volume',
  3160. 'volume_id': volume_id, 'device_name': '/dev/vdb',
  3161. 'instance_uuid': instance.uuid,
  3162. 'connection_info': '{"test": "test"}'})
  3163. vol_bdm.attachment_id = uuids.attachment
  3164. @mock.patch.object(self.compute.volume_api, 'terminate_connection')
  3165. @mock.patch.object(self.compute, 'driver')
  3166. @mock.patch.object(driver_bdm_volume, 'driver_detach')
  3167. @mock.patch.object(objects.BlockDeviceMapping,
  3168. 'get_by_volume_and_instance')
  3169. def _test(mock_get_bdms, mock_detach, mock_driver, mock_terminate):
  3170. mock_get_bdms.return_value = vol_bdm
  3171. self.compute.remove_volume_connection(self.context,
  3172. volume_id, instance)
  3173. mock_detach.assert_called_once_with(self.context, instance,
  3174. self.compute.volume_api,
  3175. mock_driver)
  3176. mock_terminate.assert_not_called()
  3177. _test()
  3178. def test_delete_disk_metadata(self):
  3179. bdm = objects.BlockDeviceMapping(volume_id=uuids.volume_id, tag='foo')
  3180. instance = fake_instance.fake_instance_obj(self.context)
  3181. instance.device_metadata = objects.InstanceDeviceMetadata(
  3182. devices=[objects.DiskMetadata(serial=uuids.volume_id,
  3183. tag='foo')])
  3184. instance.save = mock.Mock()
  3185. self.compute._delete_disk_metadata(instance, bdm)
  3186. self.assertEqual(0, len(instance.device_metadata.devices))
  3187. instance.save.assert_called_once_with()
  3188. def test_delete_disk_metadata_no_serial(self):
  3189. bdm = objects.BlockDeviceMapping(tag='foo')
  3190. instance = fake_instance.fake_instance_obj(self.context)
  3191. instance.device_metadata = objects.InstanceDeviceMetadata(
  3192. devices=[objects.DiskMetadata(tag='foo')])
  3193. self.compute._delete_disk_metadata(instance, bdm)
  3194. # NOTE(artom) This looks weird because we haven't deleted anything, but
  3195. # it's normal behaviour for when DiskMetadata doesn't have serial set
  3196. # and we can't find it based on BlockDeviceMapping's volume_id.
  3197. self.assertEqual(1, len(instance.device_metadata.devices))
  3198. def test_detach_volume(self):
  3199. # TODO(lyarwood): Test DriverVolumeBlockDevice.detach in
  3200. # ../virt/test_block_device.py
  3201. self._test_detach_volume()
  3202. def test_detach_volume_not_destroy_bdm(self):
  3203. # TODO(lyarwood): Test DriverVolumeBlockDevice.detach in
  3204. # ../virt/test_block_device.py
  3205. self._test_detach_volume(destroy_bdm=False)
  3206. @mock.patch('nova.compute.manager.ComputeManager.'
  3207. '_notify_about_instance_usage')
  3208. @mock.patch.object(driver_bdm_volume, 'detach')
  3209. @mock.patch('nova.compute.utils.notify_about_volume_attach_detach')
  3210. @mock.patch('nova.compute.manager.ComputeManager._delete_disk_metadata')
  3211. def test_detach_untagged_volume_metadata_not_deleted(
  3212. self, mock_delete_metadata, _, __, ___):
  3213. inst_obj = mock.Mock()
  3214. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  3215. {'source_type': 'volume', 'destination_type': 'volume',
  3216. 'volume_id': uuids.volume, 'device_name': '/dev/vdb',
  3217. 'connection_info': '{"test": "test"}'})
  3218. bdm = objects.BlockDeviceMapping(context=self.context, **fake_bdm)
  3219. self.compute._detach_volume(self.context, bdm, inst_obj,
  3220. destroy_bdm=False,
  3221. attachment_id=uuids.attachment)
  3222. self.assertFalse(mock_delete_metadata.called)
  3223. @mock.patch('nova.compute.manager.ComputeManager.'
  3224. '_notify_about_instance_usage')
  3225. @mock.patch.object(driver_bdm_volume, 'detach')
  3226. @mock.patch('nova.compute.utils.notify_about_volume_attach_detach')
  3227. @mock.patch('nova.compute.manager.ComputeManager._delete_disk_metadata')
  3228. def test_detach_tagged_volume(self, mock_delete_metadata, _, __, ___):
  3229. inst_obj = mock.Mock()
  3230. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  3231. {'source_type': 'volume', 'destination_type': 'volume',
  3232. 'volume_id': uuids.volume, 'device_name': '/dev/vdb',
  3233. 'connection_info': '{"test": "test"}', 'tag': 'foo'})
  3234. bdm = objects.BlockDeviceMapping(context=self.context, **fake_bdm)
  3235. self.compute._detach_volume(self.context, bdm, inst_obj,
  3236. destroy_bdm=False,
  3237. attachment_id=uuids.attachment)
  3238. mock_delete_metadata.assert_called_once_with(inst_obj, bdm)
  3239. @mock.patch.object(driver_bdm_volume, 'detach')
  3240. @mock.patch('nova.compute.manager.ComputeManager.'
  3241. '_notify_about_instance_usage')
  3242. @mock.patch('nova.compute.utils.notify_about_volume_attach_detach')
  3243. def _test_detach_volume(self, mock_notify_attach_detach, notify_inst_usage,
  3244. detach, destroy_bdm=True):
  3245. # TODO(lyarwood): Test DriverVolumeBlockDevice.detach in
  3246. # ../virt/test_block_device.py
  3247. volume_id = uuids.volume
  3248. inst_obj = mock.Mock()
  3249. inst_obj.uuid = uuids.instance
  3250. inst_obj.host = CONF.host
  3251. attachment_id = uuids.attachment
  3252. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  3253. {'source_type': 'volume', 'destination_type': 'volume',
  3254. 'volume_id': volume_id, 'device_name': '/dev/vdb',
  3255. 'connection_info': '{"test": "test"}'})
  3256. bdm = objects.BlockDeviceMapping(context=self.context, **fake_bdm)
  3257. with test.nested(
  3258. mock.patch.object(self.compute, 'volume_api'),
  3259. mock.patch.object(self.compute, 'driver'),
  3260. mock.patch.object(bdm, 'destroy'),
  3261. ) as (volume_api, driver, bdm_destroy):
  3262. self.compute._detach_volume(self.context, bdm, inst_obj,
  3263. destroy_bdm=destroy_bdm,
  3264. attachment_id=attachment_id)
  3265. detach.assert_called_once_with(self.context, inst_obj,
  3266. self.compute.volume_api, self.compute.driver,
  3267. attachment_id=attachment_id,
  3268. destroy_bdm=destroy_bdm)
  3269. notify_inst_usage.assert_called_once_with(
  3270. self.context, inst_obj, "volume.detach",
  3271. extra_usage_info={'volume_id': volume_id})
  3272. if destroy_bdm:
  3273. bdm_destroy.assert_called_once_with()
  3274. else:
  3275. self.assertFalse(bdm_destroy.called)
  3276. mock_notify_attach_detach.assert_has_calls([
  3277. mock.call(self.context, inst_obj, 'fake-mini',
  3278. action='volume_detach', phase='start',
  3279. volume_id=volume_id),
  3280. mock.call(self.context, inst_obj, 'fake-mini',
  3281. action='volume_detach', phase='end',
  3282. volume_id=volume_id),
  3283. ])
  3284. def test_detach_volume_evacuate(self):
  3285. """For evacuate, terminate_connection is called with original host."""
  3286. # TODO(lyarwood): Test DriverVolumeBlockDevice.driver_detach in
  3287. # ../virt/test_block_device.py
  3288. expected_connector = {'host': 'evacuated-host'}
  3289. conn_info_str = '{"connector": {"host": "evacuated-host"}}'
  3290. self._test_detach_volume_evacuate(conn_info_str,
  3291. expected=expected_connector)
  3292. def test_detach_volume_evacuate_legacy(self):
  3293. """Test coverage for evacuate with legacy attachments.
  3294. In this case, legacy means the volume was attached to the instance
  3295. before nova stashed the connector in connection_info. The connector
  3296. sent to terminate_connection will still be for the local host in this
  3297. case because nova does not have the info to get the connector for the
  3298. original (evacuated) host.
  3299. """
  3300. # TODO(lyarwood): Test DriverVolumeBlockDevice.driver_detach in
  3301. # ../virt/test_block_device.py
  3302. conn_info_str = '{"foo": "bar"}' # Has no 'connector'.
  3303. self._test_detach_volume_evacuate(conn_info_str)
  3304. def test_detach_volume_evacuate_mismatch(self):
  3305. """Test coverage for evacuate with connector mismatch.
  3306. For evacuate, if the stashed connector also has the wrong host,
  3307. then log it and stay with the local connector.
  3308. """
  3309. # TODO(lyarwood): Test DriverVolumeBlockDevice.driver_detach in
  3310. # ../virt/test_block_device.py
  3311. conn_info_str = '{"connector": {"host": "other-host"}}'
  3312. self._test_detach_volume_evacuate(conn_info_str)
  3313. @mock.patch('nova.compute.manager.ComputeManager.'
  3314. '_notify_about_instance_usage')
  3315. @mock.patch('nova.compute.utils.notify_about_volume_attach_detach')
  3316. def _test_detach_volume_evacuate(self, conn_info_str,
  3317. mock_notify_attach_detach,
  3318. notify_inst_usage,
  3319. expected=None):
  3320. """Re-usable code for detach volume evacuate test cases.
  3321. :param conn_info_str: String form of the stashed connector.
  3322. :param expected: Dict of the connector that is expected in the
  3323. terminate call (optional). Default is to expect the
  3324. local connector to be used.
  3325. """
  3326. # TODO(lyarwood): Test DriverVolumeBlockDevice.driver_detach in
  3327. # ../virt/test_block_device.py
  3328. volume_id = 'vol_id'
  3329. instance = fake_instance.fake_instance_obj(self.context,
  3330. host='evacuated-host')
  3331. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  3332. {'source_type': 'volume', 'destination_type': 'volume',
  3333. 'volume_id': volume_id, 'device_name': '/dev/vdb',
  3334. 'connection_info': '{"test": "test"}'})
  3335. bdm = objects.BlockDeviceMapping(context=self.context, **fake_bdm)
  3336. bdm.connection_info = conn_info_str
  3337. local_connector = {'host': 'local-connector-host'}
  3338. expected_connector = local_connector if not expected else expected
  3339. with test.nested(
  3340. mock.patch.object(self.compute, 'volume_api'),
  3341. mock.patch.object(self.compute, 'driver'),
  3342. mock.patch.object(driver_bdm_volume, 'driver_detach'),
  3343. ) as (volume_api, driver, driver_detach):
  3344. driver.get_volume_connector.return_value = local_connector
  3345. self.compute._detach_volume(self.context,
  3346. bdm,
  3347. instance,
  3348. destroy_bdm=False)
  3349. driver_detach.assert_not_called()
  3350. driver.get_volume_connector.assert_called_once_with(instance)
  3351. volume_api.terminate_connection.assert_called_once_with(
  3352. self.context, volume_id, expected_connector)
  3353. volume_api.detach.assert_called_once_with(mock.ANY,
  3354. volume_id,
  3355. instance.uuid,
  3356. None)
  3357. notify_inst_usage.assert_called_once_with(
  3358. self.context, instance, "volume.detach",
  3359. extra_usage_info={'volume_id': volume_id}
  3360. )
  3361. mock_notify_attach_detach.assert_has_calls([
  3362. mock.call(self.context, instance, 'fake-mini',
  3363. action='volume_detach', phase='start',
  3364. volume_id=volume_id),
  3365. mock.call(self.context, instance, 'fake-mini',
  3366. action='volume_detach', phase='end',
  3367. volume_id=volume_id),
  3368. ])
  3369. @mock.patch('nova.compute.utils.notify_about_instance_rescue_action')
  3370. def _test_rescue(self, mock_notify, clean_shutdown=True):
  3371. instance = fake_instance.fake_instance_obj(
  3372. self.context, vm_state=vm_states.ACTIVE)
  3373. fake_nw_info = network_model.NetworkInfo()
  3374. rescue_image_meta = objects.ImageMeta.from_dict(
  3375. {'id': uuids.image_id, 'name': uuids.image_name})
  3376. with test.nested(
  3377. mock.patch.object(self.context, 'elevated',
  3378. return_value=self.context),
  3379. mock.patch.object(self.compute.network_api, 'get_instance_nw_info',
  3380. return_value=fake_nw_info),
  3381. mock.patch.object(self.compute, '_get_rescue_image',
  3382. return_value=rescue_image_meta),
  3383. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  3384. mock.patch.object(self.compute, '_power_off_instance'),
  3385. mock.patch.object(self.compute.driver, 'rescue'),
  3386. mock.patch.object(compute_utils, 'notify_usage_exists'),
  3387. mock.patch.object(self.compute, '_get_power_state',
  3388. return_value=power_state.RUNNING),
  3389. mock.patch.object(instance, 'save')
  3390. ) as (
  3391. elevated_context, get_nw_info, get_rescue_image,
  3392. notify_instance_usage, power_off_instance, driver_rescue,
  3393. notify_usage_exists, get_power_state, instance_save
  3394. ):
  3395. self.compute.rescue_instance(
  3396. self.context, instance, rescue_password='verybadpass',
  3397. rescue_image_ref=None, clean_shutdown=clean_shutdown)
  3398. # assert the field values on the instance object
  3399. self.assertEqual(vm_states.RESCUED, instance.vm_state)
  3400. self.assertIsNone(instance.task_state)
  3401. self.assertEqual(power_state.RUNNING, instance.power_state)
  3402. self.assertIsNotNone(instance.launched_at)
  3403. # assert our mock calls
  3404. get_nw_info.assert_called_once_with(self.context, instance)
  3405. get_rescue_image.assert_called_once_with(
  3406. self.context, instance, None)
  3407. extra_usage_info = {'rescue_image_name': uuids.image_name}
  3408. notify_calls = [
  3409. mock.call(self.context, instance, "rescue.start",
  3410. extra_usage_info=extra_usage_info,
  3411. network_info=fake_nw_info),
  3412. mock.call(self.context, instance, "rescue.end",
  3413. extra_usage_info=extra_usage_info,
  3414. network_info=fake_nw_info)
  3415. ]
  3416. notify_instance_usage.assert_has_calls(notify_calls)
  3417. power_off_instance.assert_called_once_with(self.context, instance,
  3418. clean_shutdown)
  3419. driver_rescue.assert_called_once_with(
  3420. self.context, instance, fake_nw_info, rescue_image_meta,
  3421. 'verybadpass')
  3422. notify_usage_exists.assert_called_once_with(self.compute.notifier,
  3423. self.context, instance, 'fake-mini', current_period=True)
  3424. mock_notify.assert_has_calls([
  3425. mock.call(self.context, instance, 'fake-mini', None,
  3426. phase='start'),
  3427. mock.call(self.context, instance, 'fake-mini', None,
  3428. phase='end')])
  3429. instance_save.assert_called_once_with(
  3430. expected_task_state=task_states.RESCUING)
  3431. def test_rescue(self):
  3432. self._test_rescue()
  3433. def test_rescue_forced_shutdown(self):
  3434. self._test_rescue(clean_shutdown=False)
  3435. @mock.patch('nova.compute.utils.notify_about_instance_action')
  3436. def test_unrescue(self, mock_notify):
  3437. instance = fake_instance.fake_instance_obj(
  3438. self.context, vm_state=vm_states.RESCUED)
  3439. fake_nw_info = network_model.NetworkInfo()
  3440. with test.nested(
  3441. mock.patch.object(self.context, 'elevated',
  3442. return_value=self.context),
  3443. mock.patch.object(self.compute.network_api, 'get_instance_nw_info',
  3444. return_value=fake_nw_info),
  3445. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  3446. mock.patch.object(self.compute.driver, 'unrescue'),
  3447. mock.patch.object(self.compute, '_get_power_state',
  3448. return_value=power_state.RUNNING),
  3449. mock.patch.object(instance, 'save')
  3450. ) as (
  3451. elevated_context, get_nw_info, notify_instance_usage,
  3452. driver_unrescue, get_power_state, instance_save
  3453. ):
  3454. self.compute.unrescue_instance(self.context, instance)
  3455. # assert the field values on the instance object
  3456. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  3457. self.assertIsNone(instance.task_state)
  3458. self.assertEqual(power_state.RUNNING, instance.power_state)
  3459. # assert our mock calls
  3460. get_nw_info.assert_called_once_with(self.context, instance)
  3461. notify_calls = [
  3462. mock.call(self.context, instance, "unrescue.start",
  3463. network_info=fake_nw_info),
  3464. mock.call(self.context, instance, "unrescue.end",
  3465. network_info=fake_nw_info)
  3466. ]
  3467. notify_instance_usage.assert_has_calls(notify_calls)
  3468. driver_unrescue.assert_called_once_with(instance, fake_nw_info)
  3469. mock_notify.assert_has_calls([
  3470. mock.call(self.context, instance, 'fake-mini',
  3471. action='unrescue', phase='start'),
  3472. mock.call(self.context, instance, 'fake-mini',
  3473. action='unrescue', phase='end')])
  3474. instance_save.assert_called_once_with(
  3475. expected_task_state=task_states.UNRESCUING)
  3476. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  3477. return_value=power_state.RUNNING)
  3478. @mock.patch.object(objects.Instance, 'save')
  3479. @mock.patch('nova.utils.generate_password', return_value='fake-pass')
  3480. def test_set_admin_password(self, gen_password_mock, instance_save_mock,
  3481. power_state_mock):
  3482. # Ensure instance can have its admin password set.
  3483. instance = fake_instance.fake_instance_obj(
  3484. self.context,
  3485. vm_state=vm_states.ACTIVE,
  3486. task_state=task_states.UPDATING_PASSWORD)
  3487. @mock.patch.object(self.context, 'elevated', return_value=self.context)
  3488. @mock.patch.object(self.compute.driver, 'set_admin_password')
  3489. def do_test(driver_mock, elevated_mock):
  3490. # call the manager method
  3491. self.compute.set_admin_password(self.context, instance, None)
  3492. # make our assertions
  3493. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  3494. self.assertIsNone(instance.task_state)
  3495. power_state_mock.assert_called_once_with(self.context, instance)
  3496. driver_mock.assert_called_once_with(instance, 'fake-pass')
  3497. instance_save_mock.assert_called_once_with(
  3498. expected_task_state=task_states.UPDATING_PASSWORD)
  3499. do_test()
  3500. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  3501. return_value=power_state.NOSTATE)
  3502. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  3503. @mock.patch.object(objects.Instance, 'save')
  3504. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  3505. def test_set_admin_password_bad_state(self, add_fault_mock,
  3506. instance_save_mock, update_mock,
  3507. power_state_mock):
  3508. # Test setting password while instance is rebuilding.
  3509. instance = fake_instance.fake_instance_obj(self.context)
  3510. with mock.patch.object(self.context, 'elevated',
  3511. return_value=self.context):
  3512. # call the manager method
  3513. self.assertRaises(exception.InstancePasswordSetFailed,
  3514. self.compute.set_admin_password,
  3515. self.context, instance, None)
  3516. # make our assertions
  3517. power_state_mock.assert_called_once_with(self.context, instance)
  3518. instance_save_mock.assert_called_once_with(
  3519. expected_task_state=task_states.UPDATING_PASSWORD)
  3520. add_fault_mock.assert_called_once_with(
  3521. self.context, instance, mock.ANY, mock.ANY)
  3522. @mock.patch('nova.utils.generate_password', return_value='fake-pass')
  3523. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  3524. return_value=power_state.RUNNING)
  3525. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  3526. @mock.patch.object(objects.Instance, 'save')
  3527. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  3528. def _do_test_set_admin_password_driver_error(self, exc,
  3529. expected_vm_state,
  3530. expected_task_state,
  3531. expected_exception,
  3532. add_fault_mock,
  3533. instance_save_mock,
  3534. update_mock,
  3535. power_state_mock,
  3536. gen_password_mock):
  3537. # Ensure expected exception is raised if set_admin_password fails.
  3538. instance = fake_instance.fake_instance_obj(
  3539. self.context,
  3540. vm_state=vm_states.ACTIVE,
  3541. task_state=task_states.UPDATING_PASSWORD)
  3542. @mock.patch.object(self.context, 'elevated', return_value=self.context)
  3543. @mock.patch.object(self.compute.driver, 'set_admin_password',
  3544. side_effect=exc)
  3545. def do_test(driver_mock, elevated_mock):
  3546. # error raised from the driver should not reveal internal
  3547. # information so a new error is raised
  3548. self.assertRaises(expected_exception,
  3549. self.compute.set_admin_password,
  3550. self.context,
  3551. instance=instance,
  3552. new_pass=None)
  3553. if expected_exception != exception.InstancePasswordSetFailed:
  3554. instance_save_mock.assert_called_once_with(
  3555. expected_task_state=task_states.UPDATING_PASSWORD)
  3556. self.assertEqual(expected_vm_state, instance.vm_state)
  3557. # check revert_task_state decorator
  3558. update_mock.assert_called_once_with(
  3559. self.context, instance, task_state=expected_task_state)
  3560. # check wrap_instance_fault decorator
  3561. add_fault_mock.assert_called_once_with(
  3562. self.context, instance, mock.ANY, mock.ANY)
  3563. do_test()
  3564. def test_set_admin_password_driver_not_authorized(self):
  3565. # Ensure expected exception is raised if set_admin_password not
  3566. # authorized.
  3567. exc = exception.Forbidden('Internal error')
  3568. expected_exception = exception.InstancePasswordSetFailed
  3569. self._do_test_set_admin_password_driver_error(
  3570. exc, vm_states.ACTIVE, None, expected_exception)
  3571. def test_set_admin_password_driver_not_implemented(self):
  3572. # Ensure expected exception is raised if set_admin_password not
  3573. # implemented by driver.
  3574. exc = NotImplementedError()
  3575. expected_exception = NotImplementedError
  3576. self._do_test_set_admin_password_driver_error(
  3577. exc, vm_states.ACTIVE, None, expected_exception)
  3578. def test_set_admin_password_driver_not_supported(self):
  3579. exc = exception.SetAdminPasswdNotSupported()
  3580. expected_exception = exception.SetAdminPasswdNotSupported
  3581. self._do_test_set_admin_password_driver_error(
  3582. exc, vm_states.ACTIVE, None, expected_exception)
  3583. def test_set_admin_password_guest_agent_no_enabled(self):
  3584. exc = exception.QemuGuestAgentNotEnabled()
  3585. expected_exception = exception.InstanceAgentNotEnabled
  3586. self._do_test_set_admin_password_driver_error(
  3587. exc, vm_states.ACTIVE, None, expected_exception)
  3588. def test_destroy_evacuated_instances_no_migrations(self):
  3589. with mock.patch(
  3590. 'nova.objects.MigrationList.get_by_filters') as migration_list:
  3591. migration_list.return_value = []
  3592. result = self.compute._destroy_evacuated_instances(self.context)
  3593. self.assertEqual({}, result)
  3594. def test_destroy_evacuated_instances(self):
  3595. our_host = self.compute.host
  3596. flavor = objects.Flavor()
  3597. instance_1 = objects.Instance(self.context, flavor=flavor)
  3598. instance_1.uuid = uuids.instance_1
  3599. instance_1.task_state = None
  3600. instance_1.vm_state = vm_states.ACTIVE
  3601. instance_1.host = 'not-' + our_host
  3602. instance_1.user_id = uuids.user_id
  3603. instance_1.project_id = uuids.project_id
  3604. instance_2 = objects.Instance(self.context, flavor=flavor)
  3605. instance_2.uuid = uuids.instance_2
  3606. instance_2.task_state = None
  3607. instance_2.vm_state = vm_states.ACTIVE
  3608. instance_2.host = 'not-' + our_host
  3609. instance_2.user_id = uuids.user_id
  3610. instance_2.project_id = uuids.project_id
  3611. instance_2.deleted = False
  3612. # Only instance 2 has a migration record
  3613. migration = objects.Migration(instance_uuid=instance_2.uuid)
  3614. # Consider the migration successful
  3615. migration.status = 'done'
  3616. migration.source_node = 'fake-node'
  3617. our_node = objects.ComputeNode(
  3618. host=our_host, uuid=uuids.our_node_uuid)
  3619. with test.nested(
  3620. mock.patch.object(self.compute, '_get_instances_on_driver',
  3621. return_value=[instance_1,
  3622. instance_2]),
  3623. mock.patch.object(self.compute.network_api, 'get_instance_nw_info',
  3624. return_value=None),
  3625. mock.patch.object(self.compute, '_get_instance_block_device_info',
  3626. return_value={}),
  3627. mock.patch.object(self.compute, '_is_instance_storage_shared',
  3628. return_value=False),
  3629. mock.patch.object(self.compute.driver, 'destroy'),
  3630. mock.patch('nova.objects.MigrationList.get_by_filters'),
  3631. mock.patch('nova.objects.Migration.save'),
  3632. mock.patch('nova.objects.ComputeNode.get_by_host_and_nodename'),
  3633. mock.patch('nova.scheduler.utils.resources_from_flavor'),
  3634. mock.patch.object(self.compute.reportclient,
  3635. 'remove_provider_from_instance_allocation')
  3636. ) as (_get_instances_on_driver, get_instance_nw_info,
  3637. _get_instance_block_device_info, _is_instance_storage_shared,
  3638. destroy, migration_list, migration_save, get_node,
  3639. get_resources, remove_allocation):
  3640. migration_list.return_value = [migration]
  3641. get_node.return_value = our_node
  3642. get_resources.return_value = mock.sentinel.resources
  3643. self.compute._destroy_evacuated_instances(self.context)
  3644. # Only instance 2 should be deleted. Instance 1 is still running
  3645. # here, but no migration from our host exists, so ignore it
  3646. destroy.assert_called_once_with(self.context, instance_2, None,
  3647. {}, True)
  3648. get_node.assert_called_once_with(
  3649. self.context, our_host, migration.source_node)
  3650. remove_allocation.assert_called_once_with(
  3651. self.context, instance_2.uuid, uuids.our_node_uuid,
  3652. uuids.user_id, uuids.project_id, mock.sentinel.resources)
  3653. def test_destroy_evacuated_instances_node_deleted(self):
  3654. our_host = self.compute.host
  3655. flavor = objects.Flavor()
  3656. instance_1 = objects.Instance(self.context, flavor=flavor)
  3657. instance_1.uuid = uuids.instance_1
  3658. instance_1.task_state = None
  3659. instance_1.vm_state = vm_states.ACTIVE
  3660. instance_1.host = 'not-' + our_host
  3661. instance_1.user_id = uuids.user_id
  3662. instance_1.project_id = uuids.project_id
  3663. instance_1.deleted = False
  3664. instance_2 = objects.Instance(self.context, flavor=flavor)
  3665. instance_2.uuid = uuids.instance_2
  3666. instance_2.task_state = None
  3667. instance_2.vm_state = vm_states.ACTIVE
  3668. instance_2.host = 'not-' + our_host
  3669. instance_2.user_id = uuids.user_id
  3670. instance_2.project_id = uuids.project_id
  3671. instance_2.deleted = False
  3672. migration_1 = objects.Migration(instance_uuid=instance_1.uuid)
  3673. # Consider the migration successful but the node was deleted while the
  3674. # compute was down
  3675. migration_1.status = 'done'
  3676. migration_1.source_node = 'deleted-node'
  3677. migration_2 = objects.Migration(instance_uuid=instance_2.uuid)
  3678. # Consider the migration successful
  3679. migration_2.status = 'done'
  3680. migration_2.source_node = 'fake-node'
  3681. our_node = objects.ComputeNode(
  3682. host=our_host, uuid=uuids.our_node_uuid)
  3683. with test.nested(
  3684. mock.patch.object(self.compute, '_get_instances_on_driver',
  3685. return_value=[instance_1,
  3686. instance_2]),
  3687. mock.patch.object(self.compute.network_api, 'get_instance_nw_info',
  3688. return_value=None),
  3689. mock.patch.object(self.compute, '_get_instance_block_device_info',
  3690. return_value={}),
  3691. mock.patch.object(self.compute, '_is_instance_storage_shared',
  3692. return_value=False),
  3693. mock.patch.object(self.compute.driver, 'destroy'),
  3694. mock.patch('nova.objects.MigrationList.get_by_filters'),
  3695. mock.patch('nova.objects.Migration.save'),
  3696. mock.patch('nova.objects.ComputeNode.get_by_host_and_nodename'),
  3697. mock.patch('nova.scheduler.utils.resources_from_flavor'),
  3698. mock.patch.object(self.compute.reportclient,
  3699. 'remove_provider_from_instance_allocation')
  3700. ) as (_get_instances_on_driver, get_instance_nw_info,
  3701. _get_instance_block_device_info, _is_instance_storage_shared,
  3702. destroy, migration_list, migration_save, get_node,
  3703. get_resources, remove_allocation):
  3704. migration_list.return_value = [migration_1, migration_2]
  3705. def fake_get_node(context, host, node):
  3706. if node == 'fake-node':
  3707. return our_node
  3708. else:
  3709. raise exception.ComputeHostNotFound(host=host)
  3710. get_node.side_effect = fake_get_node
  3711. get_resources.return_value = mock.sentinel.resources
  3712. self.compute._destroy_evacuated_instances(self.context)
  3713. # both instance_1 and instance_2 is destroyed in the driver
  3714. destroy.assert_has_calls(
  3715. [mock.call(self.context, instance_1, None, {}, True),
  3716. mock.call(self.context, instance_2, None, {}, True)],
  3717. any_order=True)
  3718. # but only instance_2 is deallocated as the compute node for
  3719. # instance_1 is already deleted
  3720. remove_allocation.assert_called_once_with(
  3721. self.context, instance_2.uuid, uuids.our_node_uuid,
  3722. uuids.user_id, uuids.project_id, mock.sentinel.resources)
  3723. self.assertEqual(2, get_node.call_count)
  3724. def test_destroy_evacuated_instances_not_on_hypervisor(self):
  3725. our_host = self.compute.host
  3726. flavor = objects.Flavor()
  3727. instance_1 = objects.Instance(self.context, flavor=flavor)
  3728. instance_1.uuid = uuids.instance_1
  3729. instance_1.task_state = None
  3730. instance_1.vm_state = vm_states.ACTIVE
  3731. instance_1.host = 'not-' + our_host
  3732. instance_1.user_id = uuids.user_id
  3733. instance_1.project_id = uuids.project_id
  3734. instance_1.deleted = False
  3735. migration_1 = objects.Migration(instance_uuid=instance_1.uuid)
  3736. migration_1.status = 'done'
  3737. migration_1.source_node = 'our-node'
  3738. our_node = objects.ComputeNode(
  3739. host=our_host, uuid=uuids.our_node_uuid)
  3740. with test.nested(
  3741. mock.patch.object(self.compute, '_get_instances_on_driver',
  3742. return_value=[]),
  3743. mock.patch.object(self.compute.driver, 'destroy'),
  3744. mock.patch('nova.objects.MigrationList.get_by_filters'),
  3745. mock.patch('nova.objects.Migration.save'),
  3746. mock.patch('nova.objects.ComputeNode.get_by_host_and_nodename'),
  3747. mock.patch('nova.scheduler.utils.resources_from_flavor'),
  3748. mock.patch.object(self.compute.reportclient,
  3749. 'remove_provider_from_instance_allocation'),
  3750. mock.patch('nova.objects.Instance.get_by_uuid')
  3751. ) as (_get_intances_on_driver, destroy, migration_list, migration_save,
  3752. get_node, get_resources, remove_allocation,
  3753. instance_get_by_uuid):
  3754. migration_list.return_value = [migration_1]
  3755. instance_get_by_uuid.return_value = instance_1
  3756. get_node.return_value = our_node
  3757. get_resources.return_value = mock.sentinel.resources
  3758. self.compute._destroy_evacuated_instances(self.context)
  3759. # nothing to be destroyed as the driver returned no instances on
  3760. # the hypervisor
  3761. self.assertFalse(destroy.called)
  3762. # but our only instance still cleaned up in placement
  3763. remove_allocation.assert_called_once_with(
  3764. self.context, instance_1.uuid, uuids.our_node_uuid,
  3765. instance_1.user_id, instance_1.project_id,
  3766. mock.sentinel.resources)
  3767. instance_get_by_uuid.assert_called_once_with(
  3768. self.context, instance_1.uuid)
  3769. get_node.assert_called_once_with(
  3770. self.context, our_host, 'our-node')
  3771. def test_destroy_evacuated_instances_not_on_hyp_and_instance_deleted(self):
  3772. migration_1 = objects.Migration(instance_uuid=uuids.instance_1)
  3773. migration_1.status = 'done'
  3774. migration_1.source_node = 'our-node'
  3775. with test.nested(
  3776. mock.patch.object(self.compute, '_get_instances_on_driver',
  3777. return_value=[]),
  3778. mock.patch.object(self.compute.driver, 'destroy'),
  3779. mock.patch('nova.objects.MigrationList.get_by_filters'),
  3780. mock.patch('nova.objects.Migration.save'),
  3781. mock.patch('nova.scheduler.utils.resources_from_flavor'),
  3782. mock.patch.object(self.compute.reportclient,
  3783. 'remove_provider_from_instance_allocation'),
  3784. mock.patch('nova.objects.Instance.get_by_uuid')
  3785. ) as (_get_instances_on_driver,
  3786. destroy, migration_list, migration_save, get_resources,
  3787. remove_allocation, instance_get_by_uuid):
  3788. migration_list.return_value = [migration_1]
  3789. instance_get_by_uuid.side_effect = exception.InstanceNotFound(
  3790. instance_id=uuids.instance_1)
  3791. self.compute._destroy_evacuated_instances(self.context)
  3792. # nothing to be destroyed as the driver returned no instances on
  3793. # the hypervisor
  3794. self.assertFalse(destroy.called)
  3795. instance_get_by_uuid.assert_called_once_with(
  3796. self.context, uuids.instance_1)
  3797. # nothing to be cleaned as the instance was deleted already
  3798. self.assertFalse(remove_allocation.called)
  3799. @mock.patch('nova.compute.manager.ComputeManager.'
  3800. '_destroy_evacuated_instances')
  3801. @mock.patch('nova.compute.manager.LOG')
  3802. def test_init_host_foreign_instance(self, mock_log, mock_destroy):
  3803. inst = mock.MagicMock()
  3804. inst.host = self.compute.host + '-alt'
  3805. self.compute._init_instance(mock.sentinel.context, inst)
  3806. self.assertFalse(inst.save.called)
  3807. self.assertTrue(mock_log.warning.called)
  3808. msg = mock_log.warning.call_args_list[0]
  3809. self.assertIn('appears to not be owned by this host', msg[0][0])
  3810. def test_init_host_pci_passthrough_whitelist_validation_failure(self):
  3811. # Tests that we fail init_host if there is a pci.passthrough_whitelist
  3812. # configured incorrectly.
  3813. self.flags(passthrough_whitelist=[
  3814. # it's invalid to specify both in the same devspec
  3815. jsonutils.dumps({'address': 'foo', 'devname': 'bar'})],
  3816. group='pci')
  3817. self.assertRaises(exception.PciDeviceInvalidDeviceName,
  3818. self.compute.init_host)
  3819. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  3820. def test_error_out_instance_on_exception_not_implemented_err(self,
  3821. inst_update_mock):
  3822. instance = fake_instance.fake_instance_obj(self.context)
  3823. def do_test():
  3824. with self.compute._error_out_instance_on_exception(
  3825. self.context, instance, instance_state=vm_states.STOPPED):
  3826. raise NotImplementedError('test')
  3827. self.assertRaises(NotImplementedError, do_test)
  3828. inst_update_mock.assert_called_once_with(
  3829. self.context, instance,
  3830. vm_state=vm_states.STOPPED, task_state=None)
  3831. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  3832. def test_error_out_instance_on_exception_inst_fault_rollback(self,
  3833. inst_update_mock):
  3834. instance = fake_instance.fake_instance_obj(self.context)
  3835. def do_test():
  3836. with self.compute._error_out_instance_on_exception(self.context,
  3837. instance):
  3838. raise exception.InstanceFaultRollback(
  3839. inner_exception=test.TestingException('test'))
  3840. self.assertRaises(test.TestingException, do_test)
  3841. inst_update_mock.assert_called_once_with(
  3842. self.context, instance,
  3843. vm_state=vm_states.ACTIVE, task_state=None)
  3844. @mock.patch('nova.compute.manager.ComputeManager.'
  3845. '_set_instance_obj_error_state')
  3846. def test_error_out_instance_on_exception_unknown_with_quotas(self,
  3847. set_error):
  3848. instance = fake_instance.fake_instance_obj(self.context)
  3849. def do_test():
  3850. with self.compute._error_out_instance_on_exception(
  3851. self.context, instance):
  3852. raise test.TestingException('test')
  3853. self.assertRaises(test.TestingException, do_test)
  3854. set_error.assert_called_once_with(self.context, instance)
  3855. @mock.patch('nova.compute.manager.ComputeManager.'
  3856. '_detach_volume')
  3857. def test_cleanup_volumes(self, mock_detach):
  3858. instance = fake_instance.fake_instance_obj(self.context)
  3859. bdm_do_not_delete_dict = fake_block_device.FakeDbBlockDeviceDict(
  3860. {'volume_id': 'fake-id1', 'source_type': 'image',
  3861. 'delete_on_termination': False})
  3862. bdm_delete_dict = fake_block_device.FakeDbBlockDeviceDict(
  3863. {'volume_id': 'fake-id2', 'source_type': 'image',
  3864. 'delete_on_termination': True})
  3865. bdms = block_device_obj.block_device_make_list(self.context,
  3866. [bdm_do_not_delete_dict, bdm_delete_dict])
  3867. with mock.patch.object(self.compute.volume_api,
  3868. 'delete') as volume_delete:
  3869. self.compute._cleanup_volumes(self.context, instance, bdms)
  3870. calls = [mock.call(self.context, bdm, instance,
  3871. destroy_bdm=bdm.delete_on_termination)
  3872. for bdm in bdms]
  3873. self.assertEqual(calls, mock_detach.call_args_list)
  3874. volume_delete.assert_called_once_with(self.context,
  3875. bdms[1].volume_id)
  3876. @mock.patch('nova.compute.manager.ComputeManager.'
  3877. '_detach_volume')
  3878. def test_cleanup_volumes_exception_do_not_raise(self, mock_detach):
  3879. instance = fake_instance.fake_instance_obj(self.context)
  3880. bdm_dict1 = fake_block_device.FakeDbBlockDeviceDict(
  3881. {'volume_id': 'fake-id1', 'source_type': 'image',
  3882. 'delete_on_termination': True})
  3883. bdm_dict2 = fake_block_device.FakeDbBlockDeviceDict(
  3884. {'volume_id': 'fake-id2', 'source_type': 'image',
  3885. 'delete_on_termination': True})
  3886. bdms = block_device_obj.block_device_make_list(self.context,
  3887. [bdm_dict1, bdm_dict2])
  3888. with mock.patch.object(self.compute.volume_api,
  3889. 'delete',
  3890. side_effect=[test.TestingException(), None]) as volume_delete:
  3891. self.compute._cleanup_volumes(self.context, instance, bdms,
  3892. raise_exc=False)
  3893. calls = [mock.call(self.context, bdm.volume_id) for bdm in bdms]
  3894. self.assertEqual(calls, volume_delete.call_args_list)
  3895. calls = [mock.call(self.context, bdm, instance,
  3896. destroy_bdm=True) for bdm in bdms]
  3897. self.assertEqual(calls, mock_detach.call_args_list)
  3898. @mock.patch('nova.compute.manager.ComputeManager.'
  3899. '_detach_volume')
  3900. def test_cleanup_volumes_exception_raise(self, mock_detach):
  3901. instance = fake_instance.fake_instance_obj(self.context)
  3902. bdm_dict1 = fake_block_device.FakeDbBlockDeviceDict(
  3903. {'volume_id': 'fake-id1', 'source_type': 'image',
  3904. 'delete_on_termination': True})
  3905. bdm_dict2 = fake_block_device.FakeDbBlockDeviceDict(
  3906. {'volume_id': 'fake-id2', 'source_type': 'image',
  3907. 'delete_on_termination': True})
  3908. bdms = block_device_obj.block_device_make_list(self.context,
  3909. [bdm_dict1, bdm_dict2])
  3910. with mock.patch.object(self.compute.volume_api,
  3911. 'delete',
  3912. side_effect=[test.TestingException(), None]) as volume_delete:
  3913. self.assertRaises(test.TestingException,
  3914. self.compute._cleanup_volumes, self.context, instance,
  3915. bdms)
  3916. calls = [mock.call(self.context, bdm.volume_id) for bdm in bdms]
  3917. self.assertEqual(calls, volume_delete.call_args_list)
  3918. calls = [mock.call(self.context, bdm, instance,
  3919. destroy_bdm=bdm.delete_on_termination)
  3920. for bdm in bdms]
  3921. self.assertEqual(calls, mock_detach.call_args_list)
  3922. @mock.patch('nova.compute.manager.ComputeManager._detach_volume',
  3923. side_effect=exception.CinderConnectionFailed(reason='idk'))
  3924. def test_cleanup_volumes_detach_fails_raise_exc(self, mock_detach):
  3925. instance = fake_instance.fake_instance_obj(self.context)
  3926. bdms = block_device_obj.block_device_make_list(
  3927. self.context,
  3928. [fake_block_device.FakeDbBlockDeviceDict(
  3929. {'volume_id': uuids.volume_id,
  3930. 'source_type': 'volume',
  3931. 'destination_type': 'volume',
  3932. 'delete_on_termination': False})])
  3933. self.assertRaises(exception.CinderConnectionFailed,
  3934. self.compute._cleanup_volumes, self.context,
  3935. instance, bdms)
  3936. mock_detach.assert_called_once_with(
  3937. self.context, bdms[0], instance, destroy_bdm=False)
  3938. def test_stop_instance_task_state_none_power_state_shutdown(self):
  3939. # Tests that stop_instance doesn't puke when the instance power_state
  3940. # is shutdown and the task_state is None.
  3941. instance = fake_instance.fake_instance_obj(
  3942. self.context, vm_state=vm_states.ACTIVE,
  3943. task_state=None, power_state=power_state.SHUTDOWN)
  3944. @mock.patch.object(self.compute, '_get_power_state',
  3945. return_value=power_state.SHUTDOWN)
  3946. @mock.patch.object(compute_utils, 'notify_about_instance_action')
  3947. @mock.patch.object(self.compute, '_notify_about_instance_usage')
  3948. @mock.patch.object(self.compute, '_power_off_instance')
  3949. @mock.patch.object(instance, 'save')
  3950. def do_test(save_mock, power_off_mock, notify_mock,
  3951. notify_action_mock, get_state_mock):
  3952. # run the code
  3953. self.compute.stop_instance(self.context, instance, True)
  3954. # assert the calls
  3955. self.assertEqual(2, get_state_mock.call_count)
  3956. notify_mock.assert_has_calls([
  3957. mock.call(self.context, instance, 'power_off.start'),
  3958. mock.call(self.context, instance, 'power_off.end')
  3959. ])
  3960. notify_action_mock.assert_has_calls([
  3961. mock.call(self.context, instance, 'fake-mini',
  3962. action='power_off', phase='start'),
  3963. mock.call(self.context, instance, 'fake-mini',
  3964. action='power_off', phase='end'),
  3965. ])
  3966. power_off_mock.assert_called_once_with(
  3967. self.context, instance, True)
  3968. save_mock.assert_called_once_with(
  3969. expected_task_state=[task_states.POWERING_OFF, None])
  3970. self.assertEqual(power_state.SHUTDOWN, instance.power_state)
  3971. self.assertIsNone(instance.task_state)
  3972. self.assertEqual(vm_states.STOPPED, instance.vm_state)
  3973. do_test()
  3974. def test_reset_network_driver_not_implemented(self):
  3975. instance = fake_instance.fake_instance_obj(self.context)
  3976. @mock.patch.object(self.compute.driver, 'reset_network',
  3977. side_effect=NotImplementedError())
  3978. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  3979. def do_test(mock_add_fault, mock_reset):
  3980. self.assertRaises(messaging.ExpectedException,
  3981. self.compute.reset_network,
  3982. self.context,
  3983. instance)
  3984. self.compute = utils.ExceptionHelper(self.compute)
  3985. self.assertRaises(NotImplementedError,
  3986. self.compute.reset_network,
  3987. self.context,
  3988. instance)
  3989. do_test()
  3990. @mock.patch.object(manager.ComputeManager, '_set_migration_status')
  3991. @mock.patch.object(manager.ComputeManager,
  3992. '_do_rebuild_instance_with_claim')
  3993. @mock.patch('nova.compute.utils.notify_about_instance_rebuild')
  3994. @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
  3995. def _test_rebuild_ex(self, instance, exc,
  3996. mock_notify_about_instance_usage,
  3997. mock_notify, mock_rebuild, mock_set,
  3998. recreate=False, scheduled_node=None):
  3999. mock_rebuild.side_effect = exc
  4000. self.compute.rebuild_instance(self.context, instance, None, None, None,
  4001. None, None, None, recreate,
  4002. False, False, None, scheduled_node, {},
  4003. None)
  4004. mock_set.assert_called_once_with(None, 'failed')
  4005. mock_notify_about_instance_usage.assert_called_once_with(
  4006. mock.ANY, instance, 'rebuild.error', fault=mock_rebuild.side_effect
  4007. )
  4008. mock_notify.assert_called_once_with(
  4009. mock.ANY, instance, 'fake-mini', phase='error', exception=exc,
  4010. bdms=None, tb=mock.ANY)
  4011. def test_rebuild_deleting(self):
  4012. instance = fake_instance.fake_instance_obj(self.context)
  4013. ex = exception.UnexpectedDeletingTaskStateError(
  4014. instance_uuid=instance.uuid, expected='expected', actual='actual')
  4015. self._test_rebuild_ex(instance, ex)
  4016. def test_rebuild_notfound(self):
  4017. instance = fake_instance.fake_instance_obj(self.context)
  4018. ex = exception.InstanceNotFound(instance_id=instance.uuid)
  4019. self._test_rebuild_ex(instance, ex)
  4020. @mock.patch('nova.compute.utils.add_instance_fault_from_exc')
  4021. @mock.patch.object(manager.ComputeManager,
  4022. '_error_out_instance_on_exception')
  4023. def test_rebuild_driver_error_same_host(self, mock_error, mock_aiffe):
  4024. instance = fake_instance.fake_instance_obj(self.context)
  4025. ex = test.TestingException('foo')
  4026. with mock.patch.object(self.compute, '_get_resource_tracker') as mrt:
  4027. self.assertRaises(test.TestingException,
  4028. self._test_rebuild_ex, instance, ex)
  4029. rt = mrt.return_value
  4030. self.assertFalse(
  4031. rt.delete_allocation_for_evacuated_instance.called)
  4032. @mock.patch('nova.context.RequestContext.elevated')
  4033. @mock.patch('nova.compute.utils.add_instance_fault_from_exc')
  4034. @mock.patch.object(manager.ComputeManager,
  4035. '_error_out_instance_on_exception')
  4036. def test_rebuild_driver_error_evacuate(self, mock_error, mock_aiffe,
  4037. mock_elevated):
  4038. mock_elevated.return_value = self.context
  4039. instance = fake_instance.fake_instance_obj(self.context)
  4040. instance.system_metadata = {}
  4041. ex = test.TestingException('foo')
  4042. with mock.patch.object(self.compute, '_get_resource_tracker') as mrt:
  4043. self.assertRaises(test.TestingException,
  4044. self._test_rebuild_ex, instance, ex,
  4045. recreate=True, scheduled_node='foo')
  4046. rt = mrt.return_value
  4047. delete_alloc = rt.delete_allocation_for_evacuated_instance
  4048. delete_alloc.assert_called_once_with(self.context, instance, 'foo',
  4049. node_type='destination')
  4050. @mock.patch('nova.context.RequestContext.elevated')
  4051. @mock.patch('nova.objects.instance.Instance.drop_migration_context')
  4052. @mock.patch('nova.objects.instance.Instance.apply_migration_context')
  4053. @mock.patch('nova.objects.instance.Instance.mutated_migration_context')
  4054. @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
  4055. @mock.patch('nova.network.neutronv2.api.API.'
  4056. 'setup_instance_network_on_host')
  4057. @mock.patch('nova.network.neutronv2.api.API.setup_networks_on_host')
  4058. @mock.patch('nova.objects.instance.Instance.save')
  4059. @mock.patch('nova.compute.utils.notify_about_instance_rebuild')
  4060. @mock.patch('nova.compute.utils.notify_about_instance_usage')
  4061. @mock.patch('nova.compute.utils.notify_usage_exists')
  4062. @mock.patch('nova.objects.instance.Instance.image_meta',
  4063. new_callable=mock.PropertyMock)
  4064. @mock.patch('nova.compute.manager.ComputeManager.'
  4065. '_validate_instance_group_policy')
  4066. @mock.patch('nova.compute.manager.ComputeManager._set_migration_status')
  4067. @mock.patch('nova.compute.resource_tracker.ResourceTracker.rebuild_claim')
  4068. def test_evacuate_late_server_group_policy_check(
  4069. self, mock_rebuild_claim, mock_set_migration_status,
  4070. mock_validate_policy, mock_image_meta, mock_notify_exists,
  4071. mock_notify_legacy, mock_notify, mock_instance_save,
  4072. mock_setup_networks, mock_setup_intance_network, mock_get_bdms,
  4073. mock_mutate_migration, mock_appy_migration, mock_drop_migration,
  4074. mock_context_elevated):
  4075. self.flags(api_servers=['http://localhost/image/v2'], group='glance')
  4076. instance = fake_instance.fake_instance_obj(self.context)
  4077. instance.trusted_certs = None
  4078. instance.info_cache = None
  4079. elevated_context = mock.Mock()
  4080. mock_context_elevated.return_value = elevated_context
  4081. request_spec = objects.RequestSpec()
  4082. request_spec.scheduler_hints = {'group': [uuids.group]}
  4083. with mock.patch.object(self.compute, 'network_api'):
  4084. self.compute.rebuild_instance(
  4085. self.context, instance, None, None, None, None, None,
  4086. None, recreate=True, on_shared_storage=None,
  4087. preserve_ephemeral=False, migration=None,
  4088. scheduled_node='fake-node',
  4089. limits={}, request_spec=request_spec)
  4090. mock_validate_policy.assert_called_once_with(
  4091. elevated_context, instance, {'group': [uuids.group]})
  4092. @mock.patch('nova.compute.utils.add_instance_fault_from_exc')
  4093. @mock.patch('nova.compute.resource_tracker.ResourceTracker.'
  4094. 'delete_allocation_for_evacuated_instance')
  4095. @mock.patch('nova.context.RequestContext.elevated')
  4096. @mock.patch('nova.objects.instance.Instance.save')
  4097. @mock.patch('nova.compute.utils.notify_about_instance_rebuild')
  4098. @mock.patch('nova.compute.utils.notify_about_instance_usage')
  4099. @mock.patch('nova.compute.manager.ComputeManager.'
  4100. '_validate_instance_group_policy')
  4101. @mock.patch('nova.compute.manager.ComputeManager._set_migration_status')
  4102. @mock.patch('nova.compute.resource_tracker.ResourceTracker.rebuild_claim')
  4103. def test_evacuate_late_server_group_policy_check_fails(
  4104. self, mock_rebuild_claim, mock_set_migration_status,
  4105. mock_validate_policy, mock_notify_legacy, mock_notify,
  4106. mock_instance_save, mock_context_elevated, mock_delete_allocation,
  4107. mock_instance_fault):
  4108. instance = fake_instance.fake_instance_obj(self.context)
  4109. instance.info_cache = None
  4110. instance.system_metadata = {}
  4111. elevated_context = mock.Mock()
  4112. mock_context_elevated.return_value = elevated_context
  4113. request_spec = objects.RequestSpec()
  4114. request_spec.scheduler_hints = {'group': [uuids.group]}
  4115. exc = exception.RescheduledException(
  4116. instance_uuid=instance.uuid, reason='policy violation')
  4117. mock_validate_policy.side_effect = exc
  4118. self.assertRaises(
  4119. exception.BuildAbortException, self.compute.rebuild_instance,
  4120. self.context, instance, None, None, None, None, None, None,
  4121. recreate=True, on_shared_storage=None, preserve_ephemeral=False,
  4122. migration=None, scheduled_node='fake-node', limits={},
  4123. request_spec=request_spec)
  4124. mock_validate_policy.assert_called_once_with(
  4125. elevated_context, instance, {'group': [uuids.group]})
  4126. mock_delete_allocation.assert_called_once_with(
  4127. elevated_context, instance, 'fake-node', node_type='destination')
  4128. mock_notify.assert_called_once_with(
  4129. elevated_context, instance, 'fake-mini', bdms=None, exception=exc,
  4130. phase='error', tb=mock.ANY)
  4131. def test_rebuild_node_not_updated_if_not_recreate(self):
  4132. node = uuidutils.generate_uuid() # ironic node uuid
  4133. instance = fake_instance.fake_instance_obj(self.context, node=node)
  4134. instance.migration_context = None
  4135. with test.nested(
  4136. mock.patch.object(self.compute, '_get_compute_info'),
  4137. mock.patch.object(self.compute, '_do_rebuild_instance_with_claim'),
  4138. mock.patch.object(objects.Instance, 'save'),
  4139. mock.patch.object(self.compute, '_set_migration_status'),
  4140. ) as (mock_get, mock_rebuild, mock_save, mock_set):
  4141. self.compute.rebuild_instance(self.context, instance, None, None,
  4142. None, None, None, None, False,
  4143. False, False, None, None, {}, None)
  4144. self.assertFalse(mock_get.called)
  4145. self.assertEqual(node, instance.node)
  4146. mock_set.assert_called_once_with(None, 'done')
  4147. def test_rebuild_node_updated_if_recreate(self):
  4148. dead_node = uuidutils.generate_uuid()
  4149. img_sys_meta = {'image_hw_numa_nodes': 1}
  4150. instance = fake_instance.fake_instance_obj(self.context,
  4151. node=dead_node)
  4152. instance.system_metadata = img_sys_meta
  4153. instance.migration_context = None
  4154. with test.nested(
  4155. mock.patch.object(self.compute, '_get_resource_tracker'),
  4156. mock.patch.object(self.compute, '_get_compute_info'),
  4157. mock.patch.object(self.compute, '_do_rebuild_instance_with_claim'),
  4158. mock.patch.object(objects.Instance, 'save'),
  4159. mock.patch.object(self.compute, '_set_migration_status'),
  4160. ) as (mock_rt, mock_get, mock_rebuild, mock_save, mock_set):
  4161. mock_get.return_value.hypervisor_hostname = 'new-node'
  4162. self.compute.rebuild_instance(self.context, instance, None, None,
  4163. None, None, None, None, True,
  4164. False, False, None, None, {}, None)
  4165. mock_get.assert_called_once_with(mock.ANY, self.compute.host)
  4166. self.assertEqual('new-node', instance.node)
  4167. mock_set.assert_called_once_with(None, 'done')
  4168. mock_rt.assert_called_once_with()
  4169. # Make sure the rebuild_claim was called with the proper image_meta
  4170. # from the instance.
  4171. mock_rebuild_claim = mock_rt.return_value.rebuild_claim
  4172. mock_rebuild_claim.assert_called_once()
  4173. self.assertIn('image_meta', mock_rebuild_claim.call_args[1])
  4174. actual_image_meta = mock_rebuild_claim.call_args[1][
  4175. 'image_meta'].properties
  4176. self.assertIn('hw_numa_nodes', actual_image_meta)
  4177. self.assertEqual(1, actual_image_meta.hw_numa_nodes)
  4178. @mock.patch.object(compute_utils, 'notify_about_instance_rebuild')
  4179. @mock.patch.object(compute_utils, 'notify_usage_exists')
  4180. @mock.patch.object(compute_utils, 'notify_about_instance_action')
  4181. @mock.patch.object(objects.ImageMeta, 'from_instance')
  4182. @mock.patch.object(objects.Instance, 'save', return_value=None)
  4183. def test_rebuild_nw_updated_if_recreate(self,
  4184. mock_save,
  4185. mock_image_ref,
  4186. mock_notify,
  4187. mock_notify_exists,
  4188. mock_notify_rebuild):
  4189. with test.nested(
  4190. mock.patch.object(self.compute,
  4191. '_notify_about_instance_usage'),
  4192. mock.patch.object(self.compute.network_api,
  4193. 'setup_networks_on_host'),
  4194. mock.patch.object(self.compute.network_api,
  4195. 'setup_instance_network_on_host'),
  4196. mock.patch.object(self.compute.network_api,
  4197. 'get_instance_nw_info'),
  4198. mock.patch.object(self.compute,
  4199. '_get_instance_block_device_info',
  4200. return_value='fake-bdminfo'),
  4201. mock.patch.object(self.compute, '_check_trusted_certs'),
  4202. ) as(
  4203. mock_notify_usage,
  4204. mock_setup,
  4205. mock_setup_inst,
  4206. mock_get_nw_info,
  4207. mock_get_blk,
  4208. mock_check_trusted_certs
  4209. ):
  4210. self.flags(group="glance", api_servers="http://127.0.0.1:9292")
  4211. instance = fake_instance.fake_instance_obj(self.context)
  4212. orig_vif = fake_network_cache_model.new_vif(
  4213. {'profile': {"pci_slot": "0000:01:00.1"}})
  4214. orig_nw_info = network_model.NetworkInfo([orig_vif])
  4215. new_vif = fake_network_cache_model.new_vif(
  4216. {'profile': {"pci_slot": "0000:02:00.1"}})
  4217. new_nw_info = network_model.NetworkInfo([new_vif])
  4218. info_cache = objects.InstanceInfoCache(network_info=orig_nw_info,
  4219. instance_uuid=instance.uuid)
  4220. instance.info_cache = info_cache
  4221. instance.task_state = task_states.REBUILDING
  4222. instance.migration_context = None
  4223. instance.numa_topology = None
  4224. instance.pci_requests = None
  4225. instance.pci_devices = None
  4226. orig_image_ref = None
  4227. image_ref = None
  4228. injected_files = []
  4229. new_pass = None
  4230. orig_sys_metadata = None
  4231. bdms = []
  4232. recreate = True
  4233. on_shared_storage = None
  4234. preserve_ephemeral = None
  4235. mock_get_nw_info.return_value = new_nw_info
  4236. self.compute._do_rebuild_instance(self.context, instance,
  4237. orig_image_ref, image_ref,
  4238. injected_files, new_pass,
  4239. orig_sys_metadata, bdms,
  4240. recreate, on_shared_storage,
  4241. preserve_ephemeral, {}, {})
  4242. mock_notify_usage.assert_has_calls(
  4243. [mock.call(self.context, instance, "rebuild.start",
  4244. extra_usage_info=mock.ANY),
  4245. mock.call(self.context, instance, "rebuild.end",
  4246. network_info=new_nw_info,
  4247. extra_usage_info=mock.ANY)])
  4248. self.assertTrue(mock_image_ref.called)
  4249. self.assertTrue(mock_save.called)
  4250. self.assertTrue(mock_notify_exists.called)
  4251. mock_setup.assert_called_once_with(self.context, instance,
  4252. mock.ANY)
  4253. mock_setup_inst.assert_called_once_with(self.context, instance,
  4254. mock.ANY, mock.ANY)
  4255. mock_get_nw_info.assert_called_once_with(self.context, instance)
  4256. def test_rebuild_default_impl(self):
  4257. def _detach(context, bdms):
  4258. # NOTE(rpodolyaka): check that instance has been powered off by
  4259. # the time we detach block devices, exact calls arguments will be
  4260. # checked below
  4261. self.assertTrue(mock_power_off.called)
  4262. self.assertFalse(mock_destroy.called)
  4263. def _attach(context, instance, bdms):
  4264. return {'block_device_mapping': 'shared_block_storage'}
  4265. def _spawn(context, instance, image_meta, injected_files,
  4266. admin_password, allocations, network_info=None,
  4267. block_device_info=None):
  4268. self.assertEqual(block_device_info['block_device_mapping'],
  4269. 'shared_block_storage')
  4270. with test.nested(
  4271. mock.patch.object(self.compute.driver, 'destroy',
  4272. return_value=None),
  4273. mock.patch.object(self.compute.driver, 'spawn',
  4274. side_effect=_spawn),
  4275. mock.patch.object(objects.Instance, 'save',
  4276. return_value=None),
  4277. mock.patch.object(self.compute, '_power_off_instance',
  4278. return_value=None)
  4279. ) as(
  4280. mock_destroy,
  4281. mock_spawn,
  4282. mock_save,
  4283. mock_power_off
  4284. ):
  4285. instance = fake_instance.fake_instance_obj(self.context)
  4286. instance.migration_context = None
  4287. instance.numa_topology = None
  4288. instance.pci_requests = None
  4289. instance.pci_devices = None
  4290. instance.device_metadata = None
  4291. instance.task_state = task_states.REBUILDING
  4292. instance.save(expected_task_state=[task_states.REBUILDING])
  4293. self.compute._rebuild_default_impl(self.context,
  4294. instance,
  4295. None,
  4296. [],
  4297. admin_password='new_pass',
  4298. bdms=[],
  4299. allocations={},
  4300. detach_block_devices=_detach,
  4301. attach_block_devices=_attach,
  4302. network_info=None,
  4303. evacuate=False,
  4304. block_device_info=None,
  4305. preserve_ephemeral=False)
  4306. self.assertTrue(mock_save.called)
  4307. self.assertTrue(mock_spawn.called)
  4308. mock_destroy.assert_called_once_with(
  4309. self.context, instance,
  4310. network_info=None, block_device_info=None)
  4311. mock_power_off.assert_called_once_with(
  4312. self.context, instance, clean_shutdown=True)
  4313. def test_do_rebuild_instance_check_trusted_certs(self):
  4314. """Tests the scenario that we're rebuilding an instance with
  4315. trusted_certs on a host that does not support trusted certs so
  4316. a BuildAbortException is raised.
  4317. """
  4318. instance = self._trusted_certs_setup_instance()
  4319. instance.system_metadata = {}
  4320. with mock.patch.dict(self.compute.driver.capabilities,
  4321. supports_trusted_certs=False):
  4322. ex = self.assertRaises(
  4323. exception.BuildAbortException,
  4324. self.compute._do_rebuild_instance,
  4325. self.context, instance, instance.image_ref,
  4326. image_ref=None, injected_files=[], new_pass=None,
  4327. orig_sys_metadata={}, bdms=objects.BlockDeviceMapping(),
  4328. evacuate=False, on_shared_storage=None,
  4329. preserve_ephemeral=False, migration=objects.Migration(),
  4330. request_spec=objects.RequestSpec())
  4331. self.assertIn('Trusted image certificates provided on host',
  4332. six.text_type(ex))
  4333. @mock.patch.object(utils, 'last_completed_audit_period',
  4334. return_value=(0, 0))
  4335. @mock.patch.object(time, 'time', side_effect=[10, 20, 21])
  4336. @mock.patch.object(objects.InstanceList, 'get_by_host', return_value=[])
  4337. @mock.patch.object(objects.BandwidthUsage, 'get_by_instance_uuid_and_mac')
  4338. @mock.patch.object(db, 'bw_usage_update')
  4339. def test_poll_bandwidth_usage(self, bw_usage_update, get_by_uuid_mac,
  4340. get_by_host, time, last_completed_audit):
  4341. bw_counters = [{'uuid': uuids.instance, 'mac_address': 'fake-mac',
  4342. 'bw_in': 1, 'bw_out': 2}]
  4343. usage = objects.BandwidthUsage()
  4344. usage.bw_in = 3
  4345. usage.bw_out = 4
  4346. usage.last_ctr_in = 0
  4347. usage.last_ctr_out = 0
  4348. self.flags(bandwidth_poll_interval=1)
  4349. get_by_uuid_mac.return_value = usage
  4350. _time = timeutils.utcnow()
  4351. bw_usage_update.return_value = {'uuid': uuids.instance, 'mac': '',
  4352. 'start_period': _time, 'last_refreshed': _time, 'bw_in': 0,
  4353. 'bw_out': 0, 'last_ctr_in': 0, 'last_ctr_out': 0, 'deleted': 0,
  4354. 'created_at': _time, 'updated_at': _time, 'deleted_at': _time}
  4355. with mock.patch.object(self.compute.driver,
  4356. 'get_all_bw_counters', return_value=bw_counters):
  4357. self.compute._poll_bandwidth_usage(self.context)
  4358. get_by_uuid_mac.assert_called_once_with(self.context,
  4359. uuids.instance, 'fake-mac',
  4360. start_period=0, use_slave=True)
  4361. # NOTE(sdague): bw_usage_update happens at some time in
  4362. # the future, so what last_refreshed is irrelevant.
  4363. bw_usage_update.assert_called_once_with(self.context,
  4364. uuids.instance,
  4365. 'fake-mac', 0, 4, 6, 1, 2,
  4366. last_refreshed=mock.ANY,
  4367. update_cells=False)
  4368. def test_reverts_task_state_instance_not_found(self):
  4369. # Tests that the reverts_task_state decorator in the compute manager
  4370. # will not trace when an InstanceNotFound is raised.
  4371. instance = objects.Instance(uuid=uuids.instance, task_state="FAKE")
  4372. instance_update_mock = mock.Mock(
  4373. side_effect=exception.InstanceNotFound(instance_id=instance.uuid))
  4374. self.compute._instance_update = instance_update_mock
  4375. log_mock = mock.Mock()
  4376. manager.LOG = log_mock
  4377. @manager.reverts_task_state
  4378. def fake_function(self, context, instance):
  4379. raise test.TestingException()
  4380. self.assertRaises(test.TestingException, fake_function,
  4381. self, self.context, instance)
  4382. self.assertFalse(log_mock.called)
  4383. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4384. 'update_instance_info')
  4385. def test_update_scheduler_instance_info(self, mock_update):
  4386. instance = objects.Instance(uuid=uuids.instance)
  4387. self.compute._update_scheduler_instance_info(self.context, instance)
  4388. self.assertEqual(mock_update.call_count, 1)
  4389. args = mock_update.call_args[0]
  4390. self.assertNotEqual(args[0], self.context)
  4391. self.assertIsInstance(args[0], self.context.__class__)
  4392. self.assertEqual(args[1], self.compute.host)
  4393. # Send a single instance; check that the method converts to an
  4394. # InstanceList
  4395. self.assertIsInstance(args[2], objects.InstanceList)
  4396. self.assertEqual(args[2].objects[0], instance)
  4397. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4398. 'delete_instance_info')
  4399. def test_delete_scheduler_instance_info(self, mock_delete):
  4400. self.compute._delete_scheduler_instance_info(self.context,
  4401. mock.sentinel.inst_uuid)
  4402. self.assertEqual(mock_delete.call_count, 1)
  4403. args = mock_delete.call_args[0]
  4404. self.assertNotEqual(args[0], self.context)
  4405. self.assertIsInstance(args[0], self.context.__class__)
  4406. self.assertEqual(args[1], self.compute.host)
  4407. self.assertEqual(args[2], mock.sentinel.inst_uuid)
  4408. @ddt.data(('vnc', 'spice', 'rdp', 'serial_console', 'mks'),
  4409. ('spice', 'vnc', 'rdp', 'serial_console', 'mks'),
  4410. ('rdp', 'vnc', 'spice', 'serial_console', 'mks'),
  4411. ('serial_console', 'vnc', 'spice', 'rdp', 'mks'),
  4412. ('mks', 'vnc', 'spice', 'rdp', 'serial_console'))
  4413. @ddt.unpack
  4414. @mock.patch('nova.objects.ConsoleAuthToken.'
  4415. 'clean_console_auths_for_instance')
  4416. def test_clean_instance_console_tokens(self, g1, g2, g3, g4, g5,
  4417. mock_clean):
  4418. # Make sure cells v1 is disabled
  4419. self.flags(enable=False, group='cells')
  4420. # Enable one of each of the console types and disable the rest
  4421. self.flags(enabled=True, group=g1)
  4422. for g in [g2, g3, g4, g5]:
  4423. self.flags(enabled=False, group=g)
  4424. instance = objects.Instance(uuid=uuids.instance)
  4425. self.compute._clean_instance_console_tokens(self.context, instance)
  4426. mock_clean.assert_called_once_with(self.context, instance.uuid)
  4427. @mock.patch('nova.objects.ConsoleAuthToken.'
  4428. 'clean_console_auths_for_instance')
  4429. def test_clean_instance_console_tokens_no_consoles_enabled(self,
  4430. mock_clean):
  4431. for g in ['vnc', 'spice', 'rdp', 'serial_console', 'mks']:
  4432. self.flags(enabled=False, group=g)
  4433. instance = objects.Instance(uuid=uuids.instance)
  4434. self.compute._clean_instance_console_tokens(self.context, instance)
  4435. mock_clean.assert_not_called()
  4436. @mock.patch('nova.objects.ConsoleAuthToken.'
  4437. 'clean_console_auths_for_instance')
  4438. def test_clean_instance_console_tokens_cells_v1_enabled(self, mock_clean):
  4439. # Enable cells v1
  4440. self.flags(enable=True, group='cells')
  4441. self.flags(enabled=True, group='vnc')
  4442. instance = objects.Instance(uuid=uuids.instance)
  4443. self.compute._clean_instance_console_tokens(self.context, instance)
  4444. mock_clean.assert_not_called()
  4445. @mock.patch('nova.objects.ConsoleAuthToken.'
  4446. 'clean_expired_console_auths_for_host')
  4447. def test_cleanup_expired_console_auth_tokens(self, mock_clean):
  4448. # Make sure cells v1 is disabled
  4449. self.flags(enable=False, group='cells')
  4450. self.compute._cleanup_expired_console_auth_tokens(self.context)
  4451. mock_clean.assert_called_once_with(self.context, self.compute.host)
  4452. # Enable cells v1
  4453. mock_clean.reset_mock()
  4454. self.flags(enable=True, group='cells')
  4455. self.compute._cleanup_expired_console_auth_tokens(self.context)
  4456. mock_clean.assert_not_called()
  4457. @mock.patch.object(nova.context.RequestContext, 'elevated')
  4458. @mock.patch.object(nova.objects.InstanceList, 'get_by_host')
  4459. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4460. 'sync_instance_info')
  4461. def test_sync_scheduler_instance_info(self, mock_sync, mock_get_by_host,
  4462. mock_elevated):
  4463. inst1 = objects.Instance(uuid=uuids.instance_1)
  4464. inst2 = objects.Instance(uuid=uuids.instance_2)
  4465. inst3 = objects.Instance(uuid=uuids.instance_3)
  4466. exp_uuids = [inst.uuid for inst in [inst1, inst2, inst3]]
  4467. mock_get_by_host.return_value = objects.InstanceList(
  4468. objects=[inst1, inst2, inst3])
  4469. fake_elevated = context.get_admin_context()
  4470. mock_elevated.return_value = fake_elevated
  4471. self.compute._sync_scheduler_instance_info(self.context)
  4472. mock_get_by_host.assert_called_once_with(
  4473. fake_elevated, self.compute.host, expected_attrs=[],
  4474. use_slave=True)
  4475. mock_sync.assert_called_once_with(fake_elevated, self.compute.host,
  4476. exp_uuids)
  4477. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4478. 'sync_instance_info')
  4479. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4480. 'delete_instance_info')
  4481. @mock.patch.object(nova.scheduler.client.SchedulerClient,
  4482. 'update_instance_info')
  4483. def test_scheduler_info_updates_off(self, mock_update, mock_delete,
  4484. mock_sync):
  4485. mgr = self.compute
  4486. mgr.send_instance_updates = False
  4487. mgr._update_scheduler_instance_info(self.context,
  4488. mock.sentinel.instance)
  4489. mgr._delete_scheduler_instance_info(self.context,
  4490. mock.sentinel.instance_uuid)
  4491. mgr._sync_scheduler_instance_info(self.context)
  4492. # None of the calls should have been made
  4493. self.assertFalse(mock_update.called)
  4494. self.assertFalse(mock_delete.called)
  4495. self.assertFalse(mock_sync.called)
  4496. def test_refresh_instance_security_rules_takes_non_object(self):
  4497. inst = objects.Instance(uuid=uuids.instance)
  4498. with mock.patch.object(self.compute.driver,
  4499. 'refresh_instance_security_rules') as mock_r:
  4500. self.compute.refresh_instance_security_rules(self.context, inst)
  4501. self.assertIsInstance(mock_r.call_args_list[0][0][0],
  4502. objects.Instance)
  4503. def test_set_instance_obj_error_state_with_clean_task_state(self):
  4504. instance = fake_instance.fake_instance_obj(self.context,
  4505. vm_state=vm_states.BUILDING, task_state=task_states.SPAWNING)
  4506. with mock.patch.object(instance, 'save'):
  4507. self.compute._set_instance_obj_error_state(self.context, instance,
  4508. clean_task_state=True)
  4509. self.assertEqual(vm_states.ERROR, instance.vm_state)
  4510. self.assertIsNone(instance.task_state)
  4511. def test_set_instance_obj_error_state_by_default(self):
  4512. instance = fake_instance.fake_instance_obj(self.context,
  4513. vm_state=vm_states.BUILDING, task_state=task_states.SPAWNING)
  4514. with mock.patch.object(instance, 'save'):
  4515. self.compute._set_instance_obj_error_state(self.context, instance)
  4516. self.assertEqual(vm_states.ERROR, instance.vm_state)
  4517. self.assertEqual(task_states.SPAWNING, instance.task_state)
  4518. @mock.patch.object(objects.Instance, 'save')
  4519. def test_instance_update(self, mock_save):
  4520. instance = objects.Instance(task_state=task_states.SCHEDULING,
  4521. vm_state=vm_states.BUILDING)
  4522. updates = {'task_state': None, 'vm_state': vm_states.ERROR}
  4523. with mock.patch.object(self.compute,
  4524. '_update_resource_tracker') as mock_rt:
  4525. self.compute._instance_update(self.context, instance, **updates)
  4526. self.assertIsNone(instance.task_state)
  4527. self.assertEqual(vm_states.ERROR, instance.vm_state)
  4528. mock_save.assert_called_once_with()
  4529. mock_rt.assert_called_once_with(self.context, instance)
  4530. def test_reset_reloads_rpcapi(self):
  4531. orig_rpc = self.compute.compute_rpcapi
  4532. with mock.patch('nova.compute.rpcapi.ComputeAPI') as mock_rpc:
  4533. self.compute.reset()
  4534. mock_rpc.assert_called_once_with()
  4535. self.assertIsNot(orig_rpc, self.compute.compute_rpcapi)
  4536. @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
  4537. @mock.patch('nova.compute.manager.ComputeManager._delete_instance')
  4538. def test_terminate_instance_no_bdm_volume_id(self, mock_delete_instance,
  4539. mock_bdm_get_by_inst):
  4540. # Tests that we refresh the bdm list if a volume bdm does not have the
  4541. # volume_id set.
  4542. instance = fake_instance.fake_instance_obj(
  4543. self.context, vm_state=vm_states.ERROR,
  4544. task_state=task_states.DELETING)
  4545. bdm = fake_block_device.FakeDbBlockDeviceDict(
  4546. {'source_type': 'snapshot', 'destination_type': 'volume',
  4547. 'instance_uuid': instance.uuid, 'device_name': '/dev/vda'})
  4548. bdms = block_device_obj.block_device_make_list(self.context, [bdm])
  4549. # since the bdms passed in don't have a volume_id, we'll go back to the
  4550. # database looking for updated versions
  4551. mock_bdm_get_by_inst.return_value = bdms
  4552. self.compute.terminate_instance(self.context, instance, bdms)
  4553. mock_bdm_get_by_inst.assert_called_once_with(
  4554. self.context, instance.uuid)
  4555. mock_delete_instance.assert_called_once_with(
  4556. self.context, instance, bdms)
  4557. @mock.patch('nova.compute.utils.notify_about_instance_action')
  4558. @mock.patch.object(nova.compute.manager.ComputeManager,
  4559. '_notify_about_instance_usage')
  4560. @mock.patch.object(compute_utils, 'EventReporter')
  4561. def test_trigger_crash_dump(self, event_mock, notify_mock,
  4562. mock_instance_action_notify):
  4563. instance = fake_instance.fake_instance_obj(
  4564. self.context, vm_state=vm_states.ACTIVE)
  4565. self.compute.trigger_crash_dump(self.context, instance)
  4566. mock_instance_action_notify.assert_has_calls([
  4567. mock.call(self.context, instance, 'fake-mini',
  4568. action='trigger_crash_dump', phase='start'),
  4569. mock.call(self.context, instance, 'fake-mini',
  4570. action='trigger_crash_dump', phase='end')])
  4571. notify_mock.assert_has_calls([
  4572. mock.call(self.context, instance, 'trigger_crash_dump.start'),
  4573. mock.call(self.context, instance, 'trigger_crash_dump.end')
  4574. ])
  4575. self.assertIsNone(instance.task_state)
  4576. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  4577. def test_instance_restore_notification(self):
  4578. inst_obj = fake_instance.fake_instance_obj(self.context,
  4579. vm_state=vm_states.SOFT_DELETED)
  4580. with test.nested(
  4581. mock.patch.object(nova.compute.utils,
  4582. 'notify_about_instance_action'),
  4583. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  4584. mock.patch.object(objects.Instance, 'save'),
  4585. mock.patch.object(self.compute.driver, 'restore')
  4586. ) as (
  4587. fake_notify, fake_usage, fake_save, fake_restore
  4588. ):
  4589. self.compute.restore_instance(self.context, inst_obj)
  4590. fake_notify.assert_has_calls([
  4591. mock.call(self.context, inst_obj, 'fake-mini',
  4592. action='restore', phase='start'),
  4593. mock.call(self.context, inst_obj, 'fake-mini',
  4594. action='restore', phase='end')])
  4595. def test_delete_image_on_error_image_not_found_ignored(self):
  4596. """Tests that we don't log an exception trace if we get a 404 when
  4597. trying to delete an image as part of the image cleanup decorator.
  4598. """
  4599. @manager.delete_image_on_error
  4600. def some_image_related_op(self, context, image_id, instance):
  4601. raise test.TestingException('oops!')
  4602. image_id = uuids.image_id
  4603. instance = objects.Instance(uuid=uuids.instance_uuid)
  4604. with mock.patch.object(manager.LOG, 'exception') as mock_log:
  4605. with mock.patch.object(
  4606. self, 'image_api', create=True) as mock_image_api:
  4607. mock_image_api.delete.side_effect = (
  4608. exception.ImageNotFound(image_id=image_id))
  4609. self.assertRaises(test.TestingException,
  4610. some_image_related_op,
  4611. self, self.context, image_id, instance)
  4612. mock_image_api.delete.assert_called_once_with(
  4613. self.context, image_id)
  4614. # make sure nothing was logged at exception level
  4615. mock_log.assert_not_called()
  4616. @mock.patch('nova.volume.cinder.API.attachment_delete')
  4617. @mock.patch('nova.volume.cinder.API.attachment_create',
  4618. return_value={'id': uuids.attachment_id})
  4619. @mock.patch('nova.objects.BlockDeviceMapping.save')
  4620. @mock.patch('nova.volume.cinder.API.terminate_connection')
  4621. def test_terminate_volume_connections(self, mock_term_conn,
  4622. mock_bdm_save,
  4623. mock_attach_create,
  4624. mock_attach_delete):
  4625. """Tests _terminate_volume_connections with cinder v2 style,
  4626. cinder v3.44 style, and non-volume BDMs.
  4627. """
  4628. bdms = objects.BlockDeviceMappingList(
  4629. objects=[
  4630. # We use two old-style BDMs to make sure we only build the
  4631. # connector object once.
  4632. objects.BlockDeviceMapping(volume_id=uuids.v2_volume_id_1,
  4633. destination_type='volume',
  4634. attachment_id=None),
  4635. objects.BlockDeviceMapping(volume_id=uuids.v2_volume_id_2,
  4636. destination_type='volume',
  4637. attachment_id=None),
  4638. objects.BlockDeviceMapping(volume_id=uuids.v3_volume_id,
  4639. destination_type='volume',
  4640. attachment_id=uuids.attach_id),
  4641. objects.BlockDeviceMapping(volume_id=None,
  4642. destination_type='local')
  4643. ])
  4644. instance = fake_instance.fake_instance_obj(
  4645. self.context, vm_state=vm_states.ACTIVE)
  4646. fake_connector = mock.sentinel.fake_connector
  4647. with mock.patch.object(self.compute.driver, 'get_volume_connector',
  4648. return_value=fake_connector) as connector_mock:
  4649. self.compute._terminate_volume_connections(
  4650. self.context, instance, bdms)
  4651. # assert we called terminate_connection twice (once per old volume bdm)
  4652. mock_term_conn.assert_has_calls([
  4653. mock.call(self.context, uuids.v2_volume_id_1, fake_connector),
  4654. mock.call(self.context, uuids.v2_volume_id_2, fake_connector)
  4655. ])
  4656. # assert we only build the connector once
  4657. connector_mock.assert_called_once_with(instance)
  4658. # assert we called delete_attachment once for the single new volume bdm
  4659. mock_attach_delete.assert_called_once_with(
  4660. self.context, uuids.attach_id)
  4661. mock_attach_create.assert_called_once_with(
  4662. self.context, uuids.v3_volume_id, instance.uuid)
  4663. def test_instance_soft_delete_notification(self):
  4664. inst_obj = fake_instance.fake_instance_obj(self.context,
  4665. vm_state=vm_states.ACTIVE)
  4666. with test.nested(
  4667. mock.patch.object(nova.compute.utils,
  4668. 'notify_about_instance_action'),
  4669. mock.patch.object(nova.compute.utils,
  4670. 'notify_about_instance_delete'),
  4671. mock.patch.object(objects.Instance, 'save'),
  4672. mock.patch.object(self.compute.driver, 'soft_delete')
  4673. ) as (fake_notify, fake_notify_usage, fake_save, fake_soft_delete):
  4674. self.compute.soft_delete_instance(self.context, inst_obj)
  4675. fake_notify.assert_has_calls([
  4676. mock.call(self.context, inst_obj, 'fake-mini',
  4677. action='soft_delete', phase='start'),
  4678. mock.call(self.context, inst_obj, 'fake-mini',
  4679. action='soft_delete', phase='end')])
  4680. def test_get_scheduler_hints(self):
  4681. # 1. No hints and no request_spec.
  4682. self.assertEqual({}, self.compute._get_scheduler_hints({}))
  4683. # 2. Hints come from the filter_properties.
  4684. hints = {'foo': 'bar'}
  4685. filter_properties = {'scheduler_hints': hints}
  4686. self.assertEqual(
  4687. hints, self.compute._get_scheduler_hints(filter_properties))
  4688. # 3. Hints come from filter_properties because reqspec is empty.
  4689. reqspec = objects.RequestSpec.from_primitives(self.context, {}, {})
  4690. self.assertEqual(
  4691. hints, self.compute._get_scheduler_hints(
  4692. filter_properties, reqspec))
  4693. # 4. Hints come from the request spec.
  4694. reqspec_hints = {'boo': 'baz'}
  4695. reqspec = objects.RequestSpec.from_primitives(
  4696. self.context, {}, {'scheduler_hints': reqspec_hints})
  4697. # The RequestSpec unconditionally stores hints as a key=list
  4698. # unlike filter_properties which just stores whatever came in from
  4699. # the API request.
  4700. expected_reqspec_hints = {'boo': ['baz']}
  4701. self.assertDictEqual(
  4702. expected_reqspec_hints, self.compute._get_scheduler_hints(
  4703. filter_properties, reqspec))
  4704. def test_notify_volume_usage_detach_no_block_stats(self):
  4705. """Tests the case that the virt driver returns None from the
  4706. block_stats() method and no notification is sent, similar to the
  4707. virt driver raising NotImplementedError.
  4708. """
  4709. self.flags(volume_usage_poll_interval=60)
  4710. fake_instance = objects.Instance()
  4711. fake_bdm = objects.BlockDeviceMapping(device_name='/dev/vda')
  4712. with mock.patch.object(self.compute.driver, 'block_stats',
  4713. return_value=None) as block_stats:
  4714. # Assert a notification isn't sent.
  4715. with mock.patch.object(self.compute.notifier, 'info',
  4716. new_callable=mock.NonCallableMock):
  4717. self.compute._notify_volume_usage_detach(
  4718. self.context, fake_instance, fake_bdm)
  4719. block_stats.assert_called_once_with(fake_instance, 'vda')
  4720. def _test_finish_revert_resize_network_migrate_finish(
  4721. self, vifs, events, migration=None):
  4722. instance = fake_instance.fake_instance_obj(self.context)
  4723. instance.info_cache = objects.InstanceInfoCache(
  4724. network_info=network_model.NetworkInfo(vifs))
  4725. if migration is None:
  4726. migration = objects.Migration(
  4727. source_compute='fake-source',
  4728. dest_compute='fake-dest')
  4729. def fake_migrate_instance_finish(context, instance, migration):
  4730. # NOTE(artom) This looks weird, but it's checking that the
  4731. # temporaty_mutation() context manager did its job.
  4732. self.assertEqual(migration.dest_compute, migration.source_compute)
  4733. with test.nested(
  4734. mock.patch.object(self.compute.virtapi,
  4735. 'wait_for_instance_event'),
  4736. mock.patch.object(self.compute.network_api,
  4737. 'migrate_instance_finish',
  4738. side_effect=fake_migrate_instance_finish)
  4739. ) as (mock_wait, mock_migrate_instance_finish):
  4740. self.compute._finish_revert_resize_network_migrate_finish(
  4741. self.context, instance, migration)
  4742. mock_wait.assert_called_once_with(
  4743. instance, events, deadline=CONF.vif_plugging_timeout,
  4744. error_callback=self.compute._neutron_failed_migration_callback)
  4745. mock_migrate_instance_finish.assert_called_once_with(
  4746. self.context, instance, migration)
  4747. def test_finish_revert_resize_network_migrate_finish_wait(self):
  4748. """Test that we wait for bind-time events if we have a hybrid-plugged
  4749. VIF.
  4750. """
  4751. self._test_finish_revert_resize_network_migrate_finish(
  4752. [network_model.VIF(id=uuids.hybrid_vif,
  4753. details={'ovs_hybrid_plug': True}),
  4754. network_model.VIF(id=uuids.normal_vif,
  4755. details={'ovs_hybrid_plug': False})],
  4756. [('network-vif-plugged', uuids.hybrid_vif)])
  4757. def test_finish_revert_resize_network_migrate_finish_same_host(self):
  4758. """Test that we're not waiting for any events if its a same host
  4759. resize revert.
  4760. """
  4761. migration = objects.Migration(
  4762. source_compute='fake-source', dest_compute='fake-source')
  4763. self._test_finish_revert_resize_network_migrate_finish(
  4764. [network_model.VIF(id=uuids.hybrid_vif,
  4765. details={'ovs_hybrid_plug': True}),
  4766. network_model.VIF(id=uuids.normal_vif,
  4767. details={'ovs_hybrid_plug': False})],
  4768. [], migration=migration
  4769. )
  4770. def test_finish_revert_resize_network_migrate_finish_dont_wait(self):
  4771. """Test that we're not waiting for any events if we don't have any
  4772. hybrid-plugged VIFs.
  4773. """
  4774. self._test_finish_revert_resize_network_migrate_finish(
  4775. [network_model.VIF(id=uuids.hybrid_vif,
  4776. details={'ovs_hybrid_plug': False}),
  4777. network_model.VIF(id=uuids.normal_vif,
  4778. details={'ovs_hybrid_plug': False})],
  4779. [])
  4780. def test_finish_revert_resize_network_migrate_finish_no_vif_timeout(self):
  4781. """Test that we're not waiting for any events if vif_plugging_timeout
  4782. is 0.
  4783. """
  4784. self.flags(vif_plugging_timeout=0)
  4785. self._test_finish_revert_resize_network_migrate_finish(
  4786. [network_model.VIF(id=uuids.hybrid_vif,
  4787. details={'ovs_hybrid_plug': True}),
  4788. network_model.VIF(id=uuids.normal_vif,
  4789. details={'ovs_hybrid_plug': True})],
  4790. [])
  4791. @mock.patch.object(utils, 'is_neutron', return_value=False)
  4792. def test_finish_revert_resize_network_migrate_finish_not_neutron(self, _):
  4793. """Test that we're not waiting for any events if we're not using
  4794. Neutron.
  4795. """
  4796. self._test_finish_revert_resize_network_migrate_finish(
  4797. [network_model.VIF(id=uuids.hybrid_vif,
  4798. details={'ovs_hybrid_plug': True}),
  4799. network_model.VIF(id=uuids.normal_vif,
  4800. details={'ovs_hybrid_plug': True})],
  4801. [])
  4802. class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
  4803. def setUp(self):
  4804. super(ComputeManagerBuildInstanceTestCase, self).setUp()
  4805. self.compute = manager.ComputeManager()
  4806. self.context = context.RequestContext(fakes.FAKE_USER_ID,
  4807. fakes.FAKE_PROJECT_ID)
  4808. self.instance = fake_instance.fake_instance_obj(self.context,
  4809. vm_state=vm_states.ACTIVE,
  4810. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  4811. self.instance.trusted_certs = None # avoid lazy-load failures
  4812. self.admin_pass = 'pass'
  4813. self.injected_files = []
  4814. self.image = {}
  4815. self.node = 'fake-node'
  4816. self.limits = {}
  4817. self.requested_networks = []
  4818. self.security_groups = []
  4819. self.block_device_mapping = []
  4820. self.filter_properties = {'retry': {'num_attempts': 1,
  4821. 'hosts': [[self.compute.host,
  4822. 'fake-node']]}}
  4823. self.useFixture(fixtures.SpawnIsSynchronousFixture())
  4824. def fake_network_info():
  4825. return network_model.NetworkInfo([{'address': '1.2.3.4'}])
  4826. self.network_info = network_model.NetworkInfoAsyncWrapper(
  4827. fake_network_info)
  4828. self.block_device_info = self.compute._prep_block_device(context,
  4829. self.instance, self.block_device_mapping)
  4830. # override tracker with a version that doesn't need the database:
  4831. fake_rt = fake_resource_tracker.FakeResourceTracker(self.compute.host,
  4832. self.compute.driver)
  4833. self.compute._resource_tracker = fake_rt
  4834. self.allocations = [{
  4835. "resource_provider": {
  4836. "uuid": uuids.rp1,
  4837. },
  4838. "resources": {
  4839. "VCPU": 1,
  4840. "MEMORY_MB": 512,
  4841. },
  4842. }]
  4843. self.mock_get_allocs = self.useFixture(
  4844. fixtures.fixtures.MockPatch(
  4845. 'nova.scheduler.client.report.SchedulerReportClient.'
  4846. 'get_allocations_for_consumer')).mock
  4847. self.mock_get_allocs.return_value = self.allocations
  4848. def _do_build_instance_update(self, mock_save, reschedule_update=False):
  4849. mock_save.return_value = self.instance
  4850. if reschedule_update:
  4851. mock_save.side_effect = (self.instance, self.instance)
  4852. @staticmethod
  4853. def _assert_build_instance_update(mock_save,
  4854. reschedule_update=False):
  4855. if reschedule_update:
  4856. mock_save.assert_has_calls([
  4857. mock.call(expected_task_state=(task_states.SCHEDULING, None)),
  4858. mock.call()])
  4859. else:
  4860. mock_save.assert_called_once_with(expected_task_state=
  4861. (task_states.SCHEDULING, None))
  4862. def _instance_action_events(self, mock_start, mock_finish):
  4863. mock_start.assert_called_once_with(self.context, self.instance.uuid,
  4864. mock.ANY, host=CONF.host, want_result=False)
  4865. mock_finish.assert_called_once_with(self.context, self.instance.uuid,
  4866. mock.ANY, exc_val=mock.ANY, exc_tb=mock.ANY, want_result=False)
  4867. @staticmethod
  4868. def _assert_build_instance_hook_called(mock_hooks, result):
  4869. # NOTE(coreywright): we want to test the return value of
  4870. # _do_build_and_run_instance, but it doesn't bubble all the way up, so
  4871. # mock the hooking, which allows us to test that too, though a little
  4872. # too intimately
  4873. mock_hooks.setdefault().run_post.assert_called_once_with(
  4874. 'build_instance', result, mock.ANY, mock.ANY, f=None)
  4875. def test_build_and_run_instance_called_with_proper_args(self):
  4876. self._test_build_and_run_instance()
  4877. def test_build_and_run_instance_with_unlimited_max_concurrent_builds(self):
  4878. self.flags(max_concurrent_builds=0)
  4879. self.compute = manager.ComputeManager()
  4880. self._test_build_and_run_instance()
  4881. @mock.patch.object(objects.InstanceActionEvent,
  4882. 'event_finish_with_failure')
  4883. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  4884. @mock.patch.object(objects.Instance, 'save')
  4885. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  4886. @mock.patch('nova.hooks._HOOKS')
  4887. def _test_build_and_run_instance(self, mock_hooks, mock_build, mock_save,
  4888. mock_start, mock_finish):
  4889. self._do_build_instance_update(mock_save)
  4890. self.compute.build_and_run_instance(self.context, self.instance,
  4891. self.image, request_spec={},
  4892. filter_properties=self.filter_properties,
  4893. injected_files=self.injected_files,
  4894. admin_password=self.admin_pass,
  4895. requested_networks=self.requested_networks,
  4896. security_groups=self.security_groups,
  4897. block_device_mapping=self.block_device_mapping, node=self.node,
  4898. limits=self.limits, host_list=fake_host_list)
  4899. self._assert_build_instance_hook_called(mock_hooks,
  4900. build_results.ACTIVE)
  4901. self._instance_action_events(mock_start, mock_finish)
  4902. self._assert_build_instance_update(mock_save)
  4903. mock_build.assert_called_once_with(self.context, self.instance,
  4904. self.image, self.injected_files, self.admin_pass,
  4905. self.requested_networks, self.security_groups,
  4906. self.block_device_mapping, self.node, self.limits,
  4907. self.filter_properties, {})
  4908. # This test when sending an icehouse compatible rpc call to juno compute
  4909. # node, NetworkRequest object can load from three items tuple.
  4910. @mock.patch.object(compute_utils, 'EventReporter')
  4911. @mock.patch('nova.objects.Instance.save')
  4912. @mock.patch('nova.compute.manager.ComputeManager._build_and_run_instance')
  4913. def test_build_and_run_instance_with_icehouse_requested_network(
  4914. self, mock_build_and_run, mock_save, mock_event):
  4915. mock_save.return_value = self.instance
  4916. self.compute.build_and_run_instance(self.context, self.instance,
  4917. self.image, request_spec={},
  4918. filter_properties=self.filter_properties,
  4919. injected_files=self.injected_files,
  4920. admin_password=self.admin_pass,
  4921. requested_networks=[objects.NetworkRequest(
  4922. network_id='fake_network_id',
  4923. address='10.0.0.1',
  4924. port_id=uuids.port_instance)],
  4925. security_groups=self.security_groups,
  4926. block_device_mapping=self.block_device_mapping, node=self.node,
  4927. limits=self.limits, host_list=fake_host_list)
  4928. requested_network = mock_build_and_run.call_args[0][5][0]
  4929. self.assertEqual('fake_network_id', requested_network.network_id)
  4930. self.assertEqual('10.0.0.1', str(requested_network.address))
  4931. self.assertEqual(uuids.port_instance, requested_network.port_id)
  4932. @mock.patch.object(objects.InstanceActionEvent,
  4933. 'event_finish_with_failure')
  4934. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  4935. @mock.patch.object(objects.Instance, 'save')
  4936. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  4937. @mock.patch.object(manager.ComputeManager, '_cleanup_volumes')
  4938. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  4939. @mock.patch.object(manager.ComputeManager,
  4940. '_nil_out_instance_obj_host_and_node')
  4941. @mock.patch.object(manager.ComputeManager, '_set_instance_obj_error_state')
  4942. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  4943. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  4944. @mock.patch('nova.hooks._HOOKS')
  4945. def test_build_abort_exception(self, mock_hooks, mock_build_run,
  4946. mock_build, mock_set, mock_nil, mock_add,
  4947. mock_clean_vol, mock_clean_net, mock_save,
  4948. mock_start, mock_finish):
  4949. self._do_build_instance_update(mock_save)
  4950. mock_build_run.side_effect = exception.BuildAbortException(reason='',
  4951. instance_uuid=self.instance.uuid)
  4952. self.compute.build_and_run_instance(self.context, self.instance,
  4953. self.image, request_spec={},
  4954. filter_properties=self.filter_properties,
  4955. injected_files=self.injected_files,
  4956. admin_password=self.admin_pass,
  4957. requested_networks=self.requested_networks,
  4958. security_groups=self.security_groups,
  4959. block_device_mapping=self.block_device_mapping, node=self.node,
  4960. limits=self.limits, host_list=fake_host_list)
  4961. self._instance_action_events(mock_start, mock_finish)
  4962. self._assert_build_instance_update(mock_save)
  4963. self._assert_build_instance_hook_called(mock_hooks,
  4964. build_results.FAILED)
  4965. mock_build_run.assert_called_once_with(self.context, self.instance,
  4966. self.image, self.injected_files, self.admin_pass,
  4967. self.requested_networks, self.security_groups,
  4968. self.block_device_mapping, self.node, self.limits,
  4969. self.filter_properties, {})
  4970. mock_clean_net.assert_called_once_with(self.context, self.instance,
  4971. self.requested_networks)
  4972. mock_clean_vol.assert_called_once_with(self.context,
  4973. self.instance, self.block_device_mapping, raise_exc=False)
  4974. mock_add.assert_called_once_with(self.context, self.instance,
  4975. mock.ANY, mock.ANY)
  4976. mock_nil.assert_called_once_with(self.instance)
  4977. mock_set.assert_called_once_with(self.context, self.instance,
  4978. clean_task_state=True)
  4979. @mock.patch.object(objects.InstanceActionEvent,
  4980. 'event_finish_with_failure')
  4981. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  4982. @mock.patch.object(objects.Instance, 'save')
  4983. @mock.patch.object(manager.ComputeManager,
  4984. '_nil_out_instance_obj_host_and_node')
  4985. @mock.patch.object(manager.ComputeManager, '_set_instance_obj_error_state')
  4986. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  4987. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  4988. @mock.patch('nova.hooks._HOOKS')
  4989. def test_rescheduled_exception(self, mock_hooks, mock_build_run,
  4990. mock_build, mock_set, mock_nil,
  4991. mock_save, mock_start, mock_finish):
  4992. self.flags(use_neutron=False)
  4993. self._do_build_instance_update(mock_save, reschedule_update=True)
  4994. mock_build_run.side_effect = exception.RescheduledException(reason='',
  4995. instance_uuid=self.instance.uuid)
  4996. with mock.patch.object(
  4997. self.compute.network_api,
  4998. 'cleanup_instance_network_on_host') as mock_clean:
  4999. self.compute.build_and_run_instance(self.context, self.instance,
  5000. self.image, request_spec={},
  5001. filter_properties=self.filter_properties,
  5002. injected_files=self.injected_files,
  5003. admin_password=self.admin_pass,
  5004. requested_networks=self.requested_networks,
  5005. security_groups=self.security_groups,
  5006. block_device_mapping=self.block_device_mapping,
  5007. node=self.node, limits=self.limits,
  5008. host_list=fake_host_list)
  5009. self._assert_build_instance_hook_called(mock_hooks,
  5010. build_results.RESCHEDULED)
  5011. self._instance_action_events(mock_start, mock_finish)
  5012. self._assert_build_instance_update(mock_save, reschedule_update=True)
  5013. mock_build_run.assert_called_once_with(self.context, self.instance,
  5014. self.image, self.injected_files, self.admin_pass,
  5015. self.requested_networks, self.security_groups,
  5016. self.block_device_mapping, self.node, self.limits,
  5017. self.filter_properties, {})
  5018. mock_clean.assert_called_once_with(self.context, self.instance,
  5019. self.compute.host)
  5020. mock_nil.assert_called_once_with(self.instance)
  5021. mock_build.assert_called_once_with(self.context,
  5022. [self.instance], self.image, self.filter_properties,
  5023. self.admin_pass, self.injected_files, self.requested_networks,
  5024. self.security_groups, self.block_device_mapping,
  5025. request_spec={}, host_lists=[fake_host_list])
  5026. @mock.patch.object(manager.ComputeManager, '_shutdown_instance')
  5027. @mock.patch.object(manager.ComputeManager, '_build_networks_for_instance')
  5028. @mock.patch.object(fake_driver.FakeDriver, 'spawn')
  5029. @mock.patch.object(objects.Instance, 'save')
  5030. @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
  5031. def test_rescheduled_exception_with_non_ascii_exception(self,
  5032. mock_notify, mock_save, mock_spawn, mock_build, mock_shutdown):
  5033. exc = exception.NovaException(u's\xe9quence')
  5034. mock_build.return_value = self.network_info
  5035. mock_spawn.side_effect = exc
  5036. self.assertRaises(exception.RescheduledException,
  5037. self.compute._build_and_run_instance,
  5038. self.context, self.instance, self.image,
  5039. self.injected_files, self.admin_pass,
  5040. self.requested_networks, self.security_groups,
  5041. self.block_device_mapping, self.node,
  5042. self.limits, self.filter_properties)
  5043. mock_save.assert_has_calls([
  5044. mock.call(),
  5045. mock.call(),
  5046. mock.call(expected_task_state='block_device_mapping'),
  5047. ])
  5048. mock_notify.assert_has_calls([
  5049. mock.call(self.context, self.instance, 'create.start',
  5050. extra_usage_info={'image_name': self.image.get('name')}),
  5051. mock.call(self.context, self.instance, 'create.error', fault=exc)])
  5052. mock_build.assert_called_once_with(self.context, self.instance,
  5053. self.requested_networks, self.security_groups)
  5054. mock_shutdown.assert_called_once_with(self.context, self.instance,
  5055. self.block_device_mapping, self.requested_networks,
  5056. try_deallocate_networks=False)
  5057. mock_spawn.assert_called_once_with(self.context, self.instance,
  5058. test.MatchType(objects.ImageMeta), self.injected_files,
  5059. self.admin_pass, self.allocations, network_info=self.network_info,
  5060. block_device_info=self.block_device_info)
  5061. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5062. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5063. @mock.patch.object(objects.Instance, 'save')
  5064. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5065. @mock.patch.object(objects.InstanceActionEvent,
  5066. 'event_finish_with_failure')
  5067. @mock.patch.object(virt_driver.ComputeDriver, 'macs_for_instance')
  5068. def test_rescheduled_exception_with_network_allocated(self,
  5069. mock_macs_for_instance, mock_event_finish,
  5070. mock_event_start, mock_ins_save,
  5071. mock_build_ins, mock_build_and_run):
  5072. self.flags(use_neutron=False)
  5073. instance = fake_instance.fake_instance_obj(self.context,
  5074. vm_state=vm_states.ACTIVE,
  5075. system_metadata={'network_allocated': 'True'},
  5076. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  5077. mock_ins_save.return_value = instance
  5078. mock_macs_for_instance.return_value = []
  5079. mock_build_and_run.side_effect = exception.RescheduledException(
  5080. reason='', instance_uuid=self.instance.uuid)
  5081. with mock.patch.object(
  5082. self.compute.network_api,
  5083. 'cleanup_instance_network_on_host') as mock_cleanup_network:
  5084. self.compute._do_build_and_run_instance(self.context, instance,
  5085. self.image, request_spec={},
  5086. filter_properties=self.filter_properties,
  5087. injected_files=self.injected_files,
  5088. admin_password=self.admin_pass,
  5089. requested_networks=self.requested_networks,
  5090. security_groups=self.security_groups,
  5091. block_device_mapping=self.block_device_mapping, node=self.node,
  5092. limits=self.limits, host_list=fake_host_list)
  5093. mock_build_and_run.assert_called_once_with(self.context,
  5094. instance,
  5095. self.image, self.injected_files, self.admin_pass,
  5096. self.requested_networks, self.security_groups,
  5097. self.block_device_mapping, self.node, self.limits,
  5098. self.filter_properties, {})
  5099. mock_cleanup_network.assert_called_once_with(
  5100. self.context, instance, self.compute.host)
  5101. mock_build_ins.assert_called_once_with(self.context,
  5102. [instance], self.image, self.filter_properties,
  5103. self.admin_pass, self.injected_files, self.requested_networks,
  5104. self.security_groups, self.block_device_mapping,
  5105. request_spec={}, host_lists=[fake_host_list])
  5106. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5107. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5108. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5109. @mock.patch.object(objects.Instance, 'save')
  5110. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5111. @mock.patch.object(objects.InstanceActionEvent,
  5112. 'event_finish_with_failure')
  5113. @mock.patch.object(virt_driver.ComputeDriver, 'macs_for_instance')
  5114. def test_rescheduled_exception_with_network_allocated_with_neutron(self,
  5115. mock_macs_for_instance, mock_event_finish, mock_event_start,
  5116. mock_ins_save, mock_build_ins, mock_cleanup_network,
  5117. mock_build_and_run):
  5118. """Tests that we always cleanup allocated networks for the instance
  5119. when using neutron and before we reschedule off the failed host.
  5120. """
  5121. instance = fake_instance.fake_instance_obj(self.context,
  5122. vm_state=vm_states.ACTIVE,
  5123. system_metadata={'network_allocated': 'True'},
  5124. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  5125. mock_ins_save.return_value = instance
  5126. mock_macs_for_instance.return_value = []
  5127. mock_build_and_run.side_effect = exception.RescheduledException(
  5128. reason='', instance_uuid=self.instance.uuid)
  5129. self.compute._do_build_and_run_instance(self.context, instance,
  5130. self.image, request_spec={},
  5131. filter_properties=self.filter_properties,
  5132. injected_files=self.injected_files,
  5133. admin_password=self.admin_pass,
  5134. requested_networks=self.requested_networks,
  5135. security_groups=self.security_groups,
  5136. block_device_mapping=self.block_device_mapping, node=self.node,
  5137. limits=self.limits, host_list=fake_host_list)
  5138. mock_build_and_run.assert_called_once_with(self.context,
  5139. instance,
  5140. self.image, self.injected_files, self.admin_pass,
  5141. self.requested_networks, self.security_groups,
  5142. self.block_device_mapping, self.node, self.limits,
  5143. self.filter_properties, {})
  5144. mock_cleanup_network.assert_called_once_with(
  5145. self.context, instance, self.requested_networks)
  5146. mock_build_ins.assert_called_once_with(self.context,
  5147. [instance], self.image, self.filter_properties,
  5148. self.admin_pass, self.injected_files, self.requested_networks,
  5149. self.security_groups, self.block_device_mapping,
  5150. request_spec={}, host_lists=[fake_host_list])
  5151. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5152. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5153. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5154. @mock.patch.object(objects.Instance, 'save')
  5155. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5156. @mock.patch.object(objects.InstanceActionEvent,
  5157. 'event_finish_with_failure')
  5158. @mock.patch.object(virt_driver.ComputeDriver, 'macs_for_instance')
  5159. def test_rescheduled_exception_with_sriov_network_allocated(self,
  5160. mock_macs_for_instance, mock_event_finish,
  5161. mock_event_start, mock_ins_save, mock_cleanup_network,
  5162. mock_build_ins, mock_build_and_run):
  5163. vif1 = fake_network_cache_model.new_vif()
  5164. vif1['id'] = '1'
  5165. vif1['vnic_type'] = network_model.VNIC_TYPE_NORMAL
  5166. vif2 = fake_network_cache_model.new_vif()
  5167. vif2['id'] = '2'
  5168. vif1['vnic_type'] = network_model.VNIC_TYPE_DIRECT
  5169. nw_info = network_model.NetworkInfo([vif1, vif2])
  5170. instance = fake_instance.fake_instance_obj(self.context,
  5171. vm_state=vm_states.ACTIVE,
  5172. system_metadata={'network_allocated': 'True'},
  5173. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  5174. info_cache = objects.InstanceInfoCache(network_info=nw_info,
  5175. instance_uuid=instance.uuid)
  5176. instance.info_cache = info_cache
  5177. mock_ins_save.return_value = instance
  5178. mock_macs_for_instance.return_value = []
  5179. mock_build_and_run.side_effect = exception.RescheduledException(
  5180. reason='', instance_uuid=self.instance.uuid)
  5181. self.compute._do_build_and_run_instance(self.context, instance,
  5182. self.image, request_spec={},
  5183. filter_properties=self.filter_properties,
  5184. injected_files=self.injected_files,
  5185. admin_password=self.admin_pass,
  5186. requested_networks=self.requested_networks,
  5187. security_groups=self.security_groups,
  5188. block_device_mapping=self.block_device_mapping, node=self.node,
  5189. limits=self.limits, host_list=fake_host_list)
  5190. mock_build_and_run.assert_called_once_with(self.context,
  5191. instance,
  5192. self.image, self.injected_files, self.admin_pass,
  5193. self.requested_networks, self.security_groups,
  5194. self.block_device_mapping, self.node, self.limits,
  5195. self.filter_properties, {})
  5196. mock_cleanup_network.assert_called_once_with(
  5197. self.context, instance, self.requested_networks)
  5198. mock_build_ins.assert_called_once_with(self.context,
  5199. [instance], self.image, self.filter_properties,
  5200. self.admin_pass, self.injected_files, self.requested_networks,
  5201. self.security_groups, self.block_device_mapping,
  5202. request_spec={}, host_lists=[fake_host_list])
  5203. @mock.patch.object(objects.InstanceActionEvent,
  5204. 'event_finish_with_failure')
  5205. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5206. @mock.patch.object(objects.Instance, 'save')
  5207. @mock.patch.object(manager.ComputeManager,
  5208. '_nil_out_instance_obj_host_and_node')
  5209. @mock.patch.object(manager.ComputeManager, '_cleanup_volumes')
  5210. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5211. @mock.patch.object(manager.ComputeManager, '_set_instance_obj_error_state')
  5212. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  5213. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5214. @mock.patch('nova.hooks._HOOKS')
  5215. def test_rescheduled_exception_without_retry(self, mock_hooks,
  5216. mock_build_run, mock_add, mock_set, mock_clean_net, mock_clean_vol,
  5217. mock_nil, mock_save, mock_start, mock_finish):
  5218. self._do_build_instance_update(mock_save)
  5219. mock_build_run.side_effect = exception.RescheduledException(reason='',
  5220. instance_uuid=self.instance.uuid)
  5221. self.compute.build_and_run_instance(self.context, self.instance,
  5222. self.image, request_spec={},
  5223. filter_properties={},
  5224. injected_files=self.injected_files,
  5225. admin_password=self.admin_pass,
  5226. requested_networks=self.requested_networks,
  5227. security_groups=self.security_groups,
  5228. block_device_mapping=self.block_device_mapping, node=self.node,
  5229. limits=self.limits, host_list=fake_host_list)
  5230. self._assert_build_instance_hook_called(mock_hooks,
  5231. build_results.FAILED)
  5232. self._instance_action_events(mock_start, mock_finish)
  5233. self._assert_build_instance_update(mock_save)
  5234. mock_build_run.assert_called_once_with(self.context, self.instance,
  5235. self.image, self.injected_files, self.admin_pass,
  5236. self.requested_networks, self.security_groups,
  5237. self.block_device_mapping, self.node, self.limits, {}, {})
  5238. mock_clean_net.assert_called_once_with(self.context, self.instance,
  5239. self.requested_networks)
  5240. mock_clean_vol.assert_called_once_with(self.context,
  5241. self.instance, self.block_device_mapping,
  5242. raise_exc=False)
  5243. mock_add.assert_called_once_with(self.context, self.instance,
  5244. mock.ANY, mock.ANY, fault_message=mock.ANY)
  5245. mock_nil.assert_called_once_with(self.instance)
  5246. mock_set.assert_called_once_with(self.context, self.instance,
  5247. clean_task_state=True)
  5248. @mock.patch.object(objects.InstanceActionEvent,
  5249. 'event_finish_with_failure')
  5250. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5251. @mock.patch.object(objects.Instance, 'save')
  5252. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5253. @mock.patch.object(manager.ComputeManager,
  5254. '_nil_out_instance_obj_host_and_node')
  5255. @mock.patch.object(fake_driver.FakeDriver,
  5256. 'deallocate_networks_on_reschedule')
  5257. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5258. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5259. @mock.patch('nova.hooks._HOOKS')
  5260. def test_rescheduled_exception_do_not_deallocate_network(self, mock_hooks,
  5261. mock_build_run, mock_build, mock_deallocate, mock_nil,
  5262. mock_clean_net, mock_save, mock_start,
  5263. mock_finish):
  5264. self.flags(use_neutron=False)
  5265. self._do_build_instance_update(mock_save, reschedule_update=True)
  5266. mock_build_run.side_effect = exception.RescheduledException(reason='',
  5267. instance_uuid=self.instance.uuid)
  5268. mock_deallocate.return_value = False
  5269. with mock.patch.object(
  5270. self.compute.network_api,
  5271. 'cleanup_instance_network_on_host') as mock_clean_inst:
  5272. self.compute.build_and_run_instance(self.context, self.instance,
  5273. self.image, request_spec={},
  5274. filter_properties=self.filter_properties,
  5275. injected_files=self.injected_files,
  5276. admin_password=self.admin_pass,
  5277. requested_networks=self.requested_networks,
  5278. security_groups=self.security_groups,
  5279. block_device_mapping=self.block_device_mapping,
  5280. node=self.node, limits=self.limits,
  5281. host_list=fake_host_list)
  5282. self._assert_build_instance_hook_called(mock_hooks,
  5283. build_results.RESCHEDULED)
  5284. self._instance_action_events(mock_start, mock_finish)
  5285. self._assert_build_instance_update(mock_save, reschedule_update=True)
  5286. mock_build_run.assert_called_once_with(self.context, self.instance,
  5287. self.image, self.injected_files, self.admin_pass,
  5288. self.requested_networks, self.security_groups,
  5289. self.block_device_mapping, self.node, self.limits,
  5290. self.filter_properties, {})
  5291. mock_deallocate.assert_called_once_with(self.instance)
  5292. mock_clean_inst.assert_called_once_with(self.context, self.instance,
  5293. self.compute.host)
  5294. mock_nil.assert_called_once_with(self.instance)
  5295. mock_build.assert_called_once_with(self.context,
  5296. [self.instance], self.image, self.filter_properties,
  5297. self.admin_pass, self.injected_files, self.requested_networks,
  5298. self.security_groups, self.block_device_mapping,
  5299. request_spec={}, host_lists=[fake_host_list])
  5300. @mock.patch.object(objects.InstanceActionEvent,
  5301. 'event_finish_with_failure')
  5302. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5303. @mock.patch.object(objects.Instance, 'save')
  5304. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5305. @mock.patch.object(manager.ComputeManager,
  5306. '_nil_out_instance_obj_host_and_node')
  5307. @mock.patch.object(fake_driver.FakeDriver,
  5308. 'deallocate_networks_on_reschedule')
  5309. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5310. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5311. @mock.patch('nova.hooks._HOOKS')
  5312. def test_rescheduled_exception_deallocate_network(self, mock_hooks,
  5313. mock_build_run, mock_build, mock_deallocate, mock_nil, mock_clean,
  5314. mock_save, mock_start, mock_finish):
  5315. self._do_build_instance_update(mock_save, reschedule_update=True)
  5316. mock_build_run.side_effect = exception.RescheduledException(reason='',
  5317. instance_uuid=self.instance.uuid)
  5318. mock_deallocate.return_value = True
  5319. self.compute.build_and_run_instance(self.context, self.instance,
  5320. self.image, request_spec={},
  5321. filter_properties=self.filter_properties,
  5322. injected_files=self.injected_files,
  5323. admin_password=self.admin_pass,
  5324. requested_networks=self.requested_networks,
  5325. security_groups=self.security_groups,
  5326. block_device_mapping=self.block_device_mapping, node=self.node,
  5327. limits=self.limits, host_list=fake_host_list)
  5328. self._assert_build_instance_hook_called(mock_hooks,
  5329. build_results.RESCHEDULED)
  5330. self._instance_action_events(mock_start, mock_finish)
  5331. self._assert_build_instance_update(mock_save, reschedule_update=True)
  5332. mock_build_run.assert_called_once_with(self.context, self.instance,
  5333. self.image, self.injected_files, self.admin_pass,
  5334. self.requested_networks, self.security_groups,
  5335. self.block_device_mapping, self.node, self.limits,
  5336. self.filter_properties, {})
  5337. mock_deallocate.assert_called_once_with(self.instance)
  5338. mock_clean.assert_called_once_with(self.context, self.instance,
  5339. self.requested_networks)
  5340. mock_nil.assert_called_once_with(self.instance)
  5341. mock_build.assert_called_once_with(self.context,
  5342. [self.instance], self.image, self.filter_properties,
  5343. self.admin_pass, self.injected_files, self.requested_networks,
  5344. self.security_groups, self.block_device_mapping,
  5345. request_spec={}, host_lists=[fake_host_list])
  5346. @mock.patch.object(objects.InstanceActionEvent,
  5347. 'event_finish_with_failure')
  5348. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5349. @mock.patch.object(objects.Instance, 'save')
  5350. @mock.patch.object(manager.ComputeManager, '_cleanup_allocated_networks')
  5351. @mock.patch.object(manager.ComputeManager, '_cleanup_volumes')
  5352. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  5353. @mock.patch.object(manager.ComputeManager,
  5354. '_nil_out_instance_obj_host_and_node')
  5355. @mock.patch.object(manager.ComputeManager, '_set_instance_obj_error_state')
  5356. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5357. @mock.patch.object(manager.ComputeManager, '_build_and_run_instance')
  5358. @mock.patch('nova.hooks._HOOKS')
  5359. def _test_build_and_run_exceptions(self, exc, mock_hooks, mock_build_run,
  5360. mock_build, mock_set, mock_nil, mock_add, mock_clean_vol,
  5361. mock_clean_net, mock_save, mock_start, mock_finish,
  5362. set_error=False, cleanup_volumes=False,
  5363. nil_out_host_and_node=False):
  5364. self._do_build_instance_update(mock_save)
  5365. mock_build_run.side_effect = exc
  5366. self.compute.build_and_run_instance(self.context, self.instance,
  5367. self.image, request_spec={},
  5368. filter_properties=self.filter_properties,
  5369. injected_files=self.injected_files,
  5370. admin_password=self.admin_pass,
  5371. requested_networks=self.requested_networks,
  5372. security_groups=self.security_groups,
  5373. block_device_mapping=self.block_device_mapping, node=self.node,
  5374. limits=self.limits, host_list=fake_host_list)
  5375. self._assert_build_instance_hook_called(mock_hooks,
  5376. build_results.FAILED)
  5377. self._instance_action_events(mock_start, mock_finish)
  5378. self._assert_build_instance_update(mock_save)
  5379. if cleanup_volumes:
  5380. mock_clean_vol.assert_called_once_with(self.context,
  5381. self.instance, self.block_device_mapping,
  5382. raise_exc=False)
  5383. if nil_out_host_and_node:
  5384. mock_nil.assert_called_once_with(self.instance)
  5385. if set_error:
  5386. mock_add.assert_called_once_with(self.context, self.instance,
  5387. mock.ANY, mock.ANY)
  5388. mock_set.assert_called_once_with(self.context,
  5389. self.instance, clean_task_state=True)
  5390. mock_build_run.assert_called_once_with(self.context, self.instance,
  5391. self.image, self.injected_files, self.admin_pass,
  5392. self.requested_networks, self.security_groups,
  5393. self.block_device_mapping, self.node, self.limits,
  5394. self.filter_properties, {})
  5395. mock_clean_net.assert_called_once_with(self.context, self.instance,
  5396. self.requested_networks)
  5397. def test_build_and_run_notfound_exception(self):
  5398. self._test_build_and_run_exceptions(exception.InstanceNotFound(
  5399. instance_id=''))
  5400. def test_build_and_run_unexpecteddeleting_exception(self):
  5401. self._test_build_and_run_exceptions(
  5402. exception.UnexpectedDeletingTaskStateError(
  5403. instance_uuid=uuids.instance, expected={}, actual={}))
  5404. @mock.patch('nova.compute.manager.LOG.error')
  5405. def test_build_and_run_buildabort_exception(self, mock_le):
  5406. self._test_build_and_run_exceptions(
  5407. exception.BuildAbortException(instance_uuid='', reason=''),
  5408. set_error=True, cleanup_volumes=True, nil_out_host_and_node=True)
  5409. mock_le.assert_called_once_with('Build of instance aborted: ',
  5410. instance=mock.ANY)
  5411. def test_build_and_run_unhandled_exception(self):
  5412. self._test_build_and_run_exceptions(test.TestingException(),
  5413. set_error=True, cleanup_volumes=True,
  5414. nil_out_host_and_node=True)
  5415. @mock.patch.object(manager.ComputeManager, '_do_build_and_run_instance')
  5416. @mock.patch('nova.compute.stats.Stats.build_failed')
  5417. def test_build_failures_reported(self, mock_failed, mock_dbari):
  5418. mock_dbari.return_value = build_results.FAILED
  5419. instance = objects.Instance(uuid=uuids.instance)
  5420. for i in range(0, 10):
  5421. self.compute.build_and_run_instance(self.context, instance, None,
  5422. None, None)
  5423. self.assertEqual(10, mock_failed.call_count)
  5424. @mock.patch.object(manager.ComputeManager, '_do_build_and_run_instance')
  5425. @mock.patch('nova.compute.stats.Stats.build_failed')
  5426. def test_build_failures_not_reported(self, mock_failed, mock_dbari):
  5427. self.flags(consecutive_build_service_disable_threshold=0,
  5428. group='compute')
  5429. mock_dbari.return_value = build_results.FAILED
  5430. instance = objects.Instance(uuid=uuids.instance)
  5431. for i in range(0, 10):
  5432. self.compute.build_and_run_instance(self.context, instance, None,
  5433. None, None)
  5434. mock_failed.assert_not_called()
  5435. @mock.patch.object(manager.ComputeManager, '_do_build_and_run_instance')
  5436. @mock.patch.object(manager.ComputeManager, '_build_failed')
  5437. @mock.patch.object(manager.ComputeManager, '_build_succeeded')
  5438. def test_transient_build_failures_no_report(self, mock_succeeded,
  5439. mock_failed,
  5440. mock_dbari):
  5441. results = [build_results.FAILED,
  5442. build_results.ACTIVE,
  5443. build_results.RESCHEDULED]
  5444. def _fake_build(*a, **k):
  5445. if results:
  5446. return results.pop(0)
  5447. else:
  5448. return build_results.ACTIVE
  5449. mock_dbari.side_effect = _fake_build
  5450. instance = objects.Instance(uuid=uuids.instance)
  5451. for i in range(0, 10):
  5452. self.compute.build_and_run_instance(self.context, instance, None,
  5453. None, None)
  5454. self.assertEqual(2, mock_failed.call_count)
  5455. self.assertEqual(8, mock_succeeded.call_count)
  5456. @mock.patch.object(manager.ComputeManager, '_do_build_and_run_instance')
  5457. @mock.patch.object(manager.ComputeManager, '_build_failed')
  5458. @mock.patch.object(manager.ComputeManager, '_build_succeeded')
  5459. def test_build_reschedules_reported(self, mock_succeeded,
  5460. mock_failed,
  5461. mock_dbari):
  5462. mock_dbari.return_value = build_results.RESCHEDULED
  5463. instance = objects.Instance(uuid=uuids.instance)
  5464. for i in range(0, 10):
  5465. self.compute.build_and_run_instance(self.context, instance, None,
  5466. None, None)
  5467. self.assertEqual(10, mock_failed.call_count)
  5468. mock_succeeded.assert_not_called()
  5469. @mock.patch.object(manager.ComputeManager, '_do_build_and_run_instance')
  5470. @mock.patch('nova.exception_wrapper._emit_exception_notification')
  5471. @mock.patch('nova.compute.utils.add_instance_fault_from_exc')
  5472. @mock.patch.object(manager.ComputeManager, '_build_failed')
  5473. @mock.patch.object(manager.ComputeManager, '_build_succeeded')
  5474. def test_build_exceptions_reported(self, mock_succeeded,
  5475. mock_failed,
  5476. mock_if, mock_notify,
  5477. mock_dbari):
  5478. mock_dbari.side_effect = test.TestingException()
  5479. instance = objects.Instance(uuid=uuids.instance,
  5480. task_state=None)
  5481. for i in range(0, 10):
  5482. self.assertRaises(test.TestingException,
  5483. self.compute.build_and_run_instance,
  5484. self.context, instance, None,
  5485. None, None)
  5486. self.assertEqual(10, mock_failed.call_count)
  5487. mock_succeeded.assert_not_called()
  5488. @mock.patch.object(manager.ComputeManager, '_shutdown_instance')
  5489. @mock.patch.object(manager.ComputeManager, '_build_networks_for_instance')
  5490. @mock.patch.object(fake_driver.FakeDriver, 'spawn')
  5491. @mock.patch.object(objects.Instance, 'save')
  5492. @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
  5493. def _test_instance_exception(self, exc, raised_exc,
  5494. mock_notify, mock_save, mock_spawn,
  5495. mock_build, mock_shutdown):
  5496. """This method test the instance related InstanceNotFound
  5497. and reschedule on exception errors. The test cases get from
  5498. arguments.
  5499. :param exc: Injected exception into the code under test
  5500. :param exception: Raised exception in test case
  5501. :param result: At end the excepted state
  5502. """
  5503. mock_build.return_value = self.network_info
  5504. mock_spawn.side_effect = exc
  5505. self.assertRaises(raised_exc,
  5506. self.compute._build_and_run_instance,
  5507. self.context, self.instance, self.image,
  5508. self.injected_files, self.admin_pass,
  5509. self.requested_networks, self.security_groups,
  5510. self.block_device_mapping, self.node,
  5511. self.limits, self.filter_properties)
  5512. mock_save.assert_has_calls([
  5513. mock.call(),
  5514. mock.call(),
  5515. mock.call(expected_task_state='block_device_mapping')])
  5516. mock_notify.assert_has_calls([
  5517. mock.call(self.context, self.instance, 'create.start',
  5518. extra_usage_info={'image_name': self.image.get('name')}),
  5519. mock.call(self.context, self.instance, 'create.error',
  5520. fault=exc)])
  5521. mock_build.assert_called_once_with(
  5522. self.context, self.instance, self.requested_networks,
  5523. self.security_groups)
  5524. mock_shutdown.assert_called_once_with(
  5525. self.context, self.instance, self.block_device_mapping,
  5526. self.requested_networks, try_deallocate_networks=False)
  5527. mock_spawn.assert_called_once_with(
  5528. self.context, self.instance, test.MatchType(objects.ImageMeta),
  5529. self.injected_files, self.admin_pass, self.allocations,
  5530. network_info=self.network_info,
  5531. block_device_info=self.block_device_info)
  5532. def test_instance_not_found(self):
  5533. got_exc = exception.InstanceNotFound(instance_id=1)
  5534. self._test_instance_exception(got_exc, exception.InstanceNotFound)
  5535. def test_reschedule_on_exception(self):
  5536. got_exc = test.TestingException()
  5537. self._test_instance_exception(got_exc, exception.RescheduledException)
  5538. def test_spawn_network_alloc_failure(self):
  5539. # Because network allocation is asynchronous, failures may not present
  5540. # themselves until the virt spawn method is called.
  5541. self._test_build_and_run_spawn_exceptions(exception.NoMoreNetworks())
  5542. def test_spawn_network_auto_alloc_failure(self):
  5543. # This isn't really a driver.spawn failure, it's a failure from
  5544. # network_api.allocate_for_instance, but testing it here is convenient.
  5545. self._test_build_and_run_spawn_exceptions(
  5546. exception.UnableToAutoAllocateNetwork(
  5547. project_id=self.context.project_id))
  5548. def test_spawn_network_fixed_ip_not_valid_on_host_failure(self):
  5549. self._test_build_and_run_spawn_exceptions(
  5550. exception.FixedIpInvalidOnHost(port_id='fake-port-id'))
  5551. def test_build_and_run_no_more_fixedips_exception(self):
  5552. self._test_build_and_run_spawn_exceptions(
  5553. exception.NoMoreFixedIps("error messge"))
  5554. def test_build_and_run_flavor_disk_smaller_image_exception(self):
  5555. self._test_build_and_run_spawn_exceptions(
  5556. exception.FlavorDiskSmallerThanImage(
  5557. flavor_size=0, image_size=1))
  5558. def test_build_and_run_flavor_disk_smaller_min_disk(self):
  5559. self._test_build_and_run_spawn_exceptions(
  5560. exception.FlavorDiskSmallerThanMinDisk(
  5561. flavor_size=0, image_min_disk=1))
  5562. def test_build_and_run_flavor_memory_too_small_exception(self):
  5563. self._test_build_and_run_spawn_exceptions(
  5564. exception.FlavorMemoryTooSmall())
  5565. def test_build_and_run_image_not_active_exception(self):
  5566. self._test_build_and_run_spawn_exceptions(
  5567. exception.ImageNotActive(image_id=self.image.get('id')))
  5568. def test_build_and_run_image_unacceptable_exception(self):
  5569. self._test_build_and_run_spawn_exceptions(
  5570. exception.ImageUnacceptable(image_id=self.image.get('id'),
  5571. reason=""))
  5572. def test_build_and_run_invalid_disk_info_exception(self):
  5573. self._test_build_and_run_spawn_exceptions(
  5574. exception.InvalidDiskInfo(reason=""))
  5575. def test_build_and_run_invalid_disk_format_exception(self):
  5576. self._test_build_and_run_spawn_exceptions(
  5577. exception.InvalidDiskFormat(disk_format=""))
  5578. def test_build_and_run_signature_verification_error(self):
  5579. self._test_build_and_run_spawn_exceptions(
  5580. cursive_exception.SignatureVerificationError(reason=""))
  5581. def test_build_and_run_certificate_validation_error(self):
  5582. self._test_build_and_run_spawn_exceptions(
  5583. exception.CertificateValidationFailed(cert_uuid='trusted-cert-id',
  5584. reason=""))
  5585. def test_build_and_run_volume_encryption_not_supported(self):
  5586. self._test_build_and_run_spawn_exceptions(
  5587. exception.VolumeEncryptionNotSupported(volume_type='something',
  5588. volume_id='something'))
  5589. def test_build_and_run_invalid_input(self):
  5590. self._test_build_and_run_spawn_exceptions(
  5591. exception.InvalidInput(reason=""))
  5592. def test_build_and_run_requested_vram_too_high(self):
  5593. self._test_build_and_run_spawn_exceptions(
  5594. exception.RequestedVRamTooHigh(req_vram=200, max_vram=100))
  5595. def _test_build_and_run_spawn_exceptions(self, exc):
  5596. with test.nested(
  5597. mock.patch.object(self.compute.driver, 'spawn',
  5598. side_effect=exc),
  5599. mock.patch.object(self.instance, 'save',
  5600. side_effect=[self.instance, self.instance, self.instance]),
  5601. mock.patch.object(self.compute,
  5602. '_build_networks_for_instance',
  5603. return_value=self.network_info),
  5604. mock.patch.object(self.compute,
  5605. '_notify_about_instance_usage'),
  5606. mock.patch.object(self.compute,
  5607. '_shutdown_instance'),
  5608. mock.patch.object(self.compute,
  5609. '_validate_instance_group_policy'),
  5610. mock.patch('nova.compute.utils.notify_about_instance_create')
  5611. ) as (spawn, save,
  5612. _build_networks_for_instance, _notify_about_instance_usage,
  5613. _shutdown_instance, _validate_instance_group_policy,
  5614. mock_notify):
  5615. self.assertRaises(exception.BuildAbortException,
  5616. self.compute._build_and_run_instance, self.context,
  5617. self.instance, self.image, self.injected_files,
  5618. self.admin_pass, self.requested_networks,
  5619. self.security_groups, self.block_device_mapping, self.node,
  5620. self.limits, self.filter_properties)
  5621. _validate_instance_group_policy.assert_called_once_with(
  5622. self.context, self.instance, {})
  5623. _build_networks_for_instance.assert_has_calls(
  5624. [mock.call(self.context, self.instance,
  5625. self.requested_networks, self.security_groups)])
  5626. _notify_about_instance_usage.assert_has_calls([
  5627. mock.call(self.context, self.instance, 'create.start',
  5628. extra_usage_info={'image_name': self.image.get('name')}),
  5629. mock.call(self.context, self.instance, 'create.error',
  5630. fault=exc)])
  5631. mock_notify.assert_has_calls([
  5632. mock.call(self.context, self.instance, 'fake-mini',
  5633. phase='start', bdms=[]),
  5634. mock.call(self.context, self.instance, 'fake-mini',
  5635. phase='error', exception=exc, bdms=[], tb=mock.ANY)])
  5636. save.assert_has_calls([
  5637. mock.call(),
  5638. mock.call(),
  5639. mock.call(
  5640. expected_task_state=task_states.BLOCK_DEVICE_MAPPING)])
  5641. spawn.assert_has_calls([mock.call(self.context, self.instance,
  5642. test.MatchType(objects.ImageMeta),
  5643. self.injected_files, self.admin_pass, self.allocations,
  5644. network_info=self.network_info,
  5645. block_device_info=self.block_device_info)])
  5646. _shutdown_instance.assert_called_once_with(self.context,
  5647. self.instance, self.block_device_mapping,
  5648. self.requested_networks, try_deallocate_networks=False)
  5649. @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
  5650. @mock.patch.object(objects.InstanceActionEvent,
  5651. 'event_finish_with_failure')
  5652. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  5653. @mock.patch.object(objects.Instance, 'save')
  5654. @mock.patch.object(manager.ComputeManager,
  5655. '_nil_out_instance_obj_host_and_node')
  5656. @mock.patch.object(conductor_api.ComputeTaskAPI, 'build_instances')
  5657. @mock.patch.object(resource_tracker.ResourceTracker, 'instance_claim')
  5658. def test_reschedule_on_resources_unavailable(self, mock_claim,
  5659. mock_build, mock_nil, mock_save, mock_start,
  5660. mock_finish, mock_notify):
  5661. self.flags(use_neutron=False)
  5662. reason = 'resource unavailable'
  5663. exc = exception.ComputeResourcesUnavailable(reason=reason)
  5664. mock_claim.side_effect = exc
  5665. self._do_build_instance_update(mock_save, reschedule_update=True)
  5666. with mock.patch.object(
  5667. self.compute.network_api,
  5668. 'cleanup_instance_network_on_host') as mock_clean:
  5669. self.compute.build_and_run_instance(self.context, self.instance,
  5670. self.image, request_spec={},
  5671. filter_properties=self.filter_properties,
  5672. injected_files=self.injected_files,
  5673. admin_password=self.admin_pass,
  5674. requested_networks=self.requested_networks,
  5675. security_groups=self.security_groups,
  5676. block_device_mapping=self.block_device_mapping,
  5677. node=self.node, limits=self.limits,
  5678. host_list=fake_host_list)
  5679. self._instance_action_events(mock_start, mock_finish)
  5680. self._assert_build_instance_update(mock_save, reschedule_update=True)
  5681. mock_claim.assert_called_once_with(self.context, self.instance,
  5682. self.node, self.limits)
  5683. mock_notify.assert_has_calls([
  5684. mock.call(self.context, self.instance, 'create.start',
  5685. extra_usage_info= {'image_name': self.image.get('name')}),
  5686. mock.call(self.context, self.instance, 'create.error', fault=exc)])
  5687. mock_build.assert_called_once_with(self.context, [self.instance],
  5688. self.image, self.filter_properties, self.admin_pass,
  5689. self.injected_files, self.requested_networks,
  5690. self.security_groups, self.block_device_mapping,
  5691. request_spec={}, host_lists=[fake_host_list])
  5692. mock_nil.assert_called_once_with(self.instance)
  5693. mock_clean.assert_called_once_with(self.context, self.instance,
  5694. self.compute.host)
  5695. @mock.patch.object(manager.ComputeManager, '_build_resources')
  5696. @mock.patch.object(objects.Instance, 'save')
  5697. @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
  5698. def test_build_resources_buildabort_reraise(self, mock_notify, mock_save,
  5699. mock_build):
  5700. exc = exception.BuildAbortException(
  5701. instance_uuid=self.instance.uuid, reason='')
  5702. mock_build.side_effect = exc
  5703. self.assertRaises(exception.BuildAbortException,
  5704. self.compute._build_and_run_instance,
  5705. self.context,
  5706. self.instance, self.image, self.injected_files,
  5707. self.admin_pass, self.requested_networks,
  5708. self.security_groups, self.block_device_mapping,
  5709. self.node, self.limits, self.filter_properties)
  5710. mock_save.assert_called_once_with()
  5711. mock_notify.assert_has_calls([
  5712. mock.call(self.context, self.instance, 'create.start',
  5713. extra_usage_info={'image_name': self.image.get('name')}),
  5714. mock.call(self.context, self.instance, 'create.error',
  5715. fault=exc)])
  5716. mock_build.assert_called_once_with(self.context, self.instance,
  5717. self.requested_networks, self.security_groups,
  5718. test.MatchType(objects.ImageMeta), self.block_device_mapping)
  5719. @mock.patch.object(virt_driver.ComputeDriver, 'failed_spawn_cleanup')
  5720. @mock.patch.object(virt_driver.ComputeDriver, 'prepare_for_spawn')
  5721. @mock.patch.object(objects.Instance, 'save')
  5722. @mock.patch.object(manager.ComputeManager, '_build_networks_for_instance')
  5723. @mock.patch.object(manager.ComputeManager, '_prep_block_device')
  5724. @mock.patch.object(virt_driver.ComputeDriver,
  5725. 'prepare_networks_before_block_device_mapping')
  5726. @mock.patch.object(virt_driver.ComputeDriver,
  5727. 'clean_networks_preparation')
  5728. def test_build_resources_reraises_on_failed_bdm_prep(
  5729. self, mock_clean, mock_prepnet, mock_prep, mock_build, mock_save,
  5730. mock_prepspawn, mock_failedspawn):
  5731. mock_save.return_value = self.instance
  5732. mock_build.return_value = self.network_info
  5733. mock_prep.side_effect = test.TestingException
  5734. try:
  5735. with self.compute._build_resources(self.context, self.instance,
  5736. self.requested_networks, self.security_groups,
  5737. self.image, self.block_device_mapping):
  5738. pass
  5739. except Exception as e:
  5740. self.assertIsInstance(e, exception.BuildAbortException)
  5741. mock_save.assert_called_once_with()
  5742. mock_build.assert_called_once_with(self.context, self.instance,
  5743. self.requested_networks, self.security_groups)
  5744. mock_prep.assert_called_once_with(self.context, self.instance,
  5745. self.block_device_mapping)
  5746. mock_prepnet.assert_called_once_with(self.instance, self.network_info)
  5747. mock_clean.assert_called_once_with(self.instance, self.network_info)
  5748. mock_prepspawn.assert_called_once_with(self.instance)
  5749. mock_failedspawn.assert_called_once_with(self.instance)
  5750. @mock.patch('nova.virt.block_device.attach_block_devices',
  5751. side_effect=exception.VolumeNotCreated('oops!'))
  5752. def test_prep_block_device_maintain_original_error_message(self,
  5753. mock_attach):
  5754. """Tests that when attach_block_devices raises an Exception, the
  5755. re-raised InvalidBDM has the original error message which contains
  5756. the actual details of the failure.
  5757. """
  5758. bdms = objects.BlockDeviceMappingList(
  5759. objects=[fake_block_device.fake_bdm_object(
  5760. self.context,
  5761. dict(source_type='image',
  5762. destination_type='volume',
  5763. boot_index=0,
  5764. image_id=uuids.image_id,
  5765. device_name='/dev/vda',
  5766. volume_size=1))])
  5767. ex = self.assertRaises(exception.InvalidBDM,