OpenStack Compute (Nova)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

3066 lines
144KB

  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. """Unit tests for ComputeManager()."""
  13. import contextlib
  14. import time
  15. from cinderclient import exceptions as cinder_exception
  16. from eventlet import event as eventlet_event
  17. import mock
  18. import mox
  19. from oslo.config import cfg
  20. from oslo import messaging
  21. from oslo.utils import importutils
  22. from nova.compute import manager
  23. from nova.compute import power_state
  24. from nova.compute import task_states
  25. from nova.compute import utils as compute_utils
  26. from nova.compute import vm_states
  27. from nova.conductor import rpcapi as conductor_rpcapi
  28. from nova import context
  29. from nova import db
  30. from nova import exception
  31. from nova.network import api as network_api
  32. from nova.network import model as network_model
  33. from nova import objects
  34. from nova.objects import block_device as block_device_obj
  35. from nova.openstack.common import uuidutils
  36. from nova import test
  37. from nova.tests.unit.compute import fake_resource_tracker
  38. from nova.tests.unit import fake_block_device
  39. from nova.tests.unit import fake_instance
  40. from nova.tests.unit.objects import test_instance_fault
  41. from nova.tests.unit.objects import test_instance_info_cache
  42. from nova import utils
  43. CONF = cfg.CONF
  44. CONF.import_opt('compute_manager', 'nova.service')
  45. class ComputeManagerUnitTestCase(test.NoDBTestCase):
  46. def setUp(self):
  47. super(ComputeManagerUnitTestCase, self).setUp()
  48. self.compute = importutils.import_object(CONF.compute_manager)
  49. self.context = context.RequestContext('fake', 'fake')
  50. def test_allocate_network_succeeds_after_retries(self):
  51. self.flags(network_allocate_retries=8)
  52. nwapi = self.compute.network_api
  53. self.mox.StubOutWithMock(nwapi, 'allocate_for_instance')
  54. self.mox.StubOutWithMock(self.compute, '_instance_update')
  55. self.mox.StubOutWithMock(time, 'sleep')
  56. instance = fake_instance.fake_instance_obj(
  57. self.context, expected_attrs=['system_metadata'])
  58. is_vpn = 'fake-is-vpn'
  59. req_networks = 'fake-req-networks'
  60. macs = 'fake-macs'
  61. sec_groups = 'fake-sec-groups'
  62. final_result = 'meow'
  63. dhcp_options = None
  64. expected_sleep_times = [1, 2, 4, 8, 16, 30, 30, 30]
  65. for sleep_time in expected_sleep_times:
  66. nwapi.allocate_for_instance(
  67. self.context, instance, vpn=is_vpn,
  68. requested_networks=req_networks, macs=macs,
  69. security_groups=sec_groups,
  70. dhcp_options=dhcp_options).AndRaise(
  71. test.TestingException())
  72. time.sleep(sleep_time)
  73. nwapi.allocate_for_instance(
  74. self.context, instance, vpn=is_vpn,
  75. requested_networks=req_networks, macs=macs,
  76. security_groups=sec_groups,
  77. dhcp_options=dhcp_options).AndReturn(final_result)
  78. self.compute._instance_update(self.context, instance['uuid'],
  79. system_metadata={'network_allocated': 'True'})
  80. self.mox.ReplayAll()
  81. res = self.compute._allocate_network_async(self.context, instance,
  82. req_networks,
  83. macs,
  84. sec_groups,
  85. is_vpn,
  86. dhcp_options)
  87. self.assertEqual(final_result, res)
  88. def test_allocate_network_maintains_context(self):
  89. # override tracker with a version that doesn't need the database:
  90. class FakeResourceTracker(object):
  91. def instance_claim(self, context, instance, limits):
  92. return mox.MockAnything()
  93. self.mox.StubOutWithMock(self.compute, '_get_resource_tracker')
  94. self.mox.StubOutWithMock(self.compute, '_allocate_network')
  95. self.mox.StubOutWithMock(objects.BlockDeviceMappingList,
  96. 'get_by_instance_uuid')
  97. instance = fake_instance.fake_instance_obj(self.context)
  98. objects.BlockDeviceMappingList.get_by_instance_uuid(
  99. mox.IgnoreArg(), instance.uuid).AndReturn([])
  100. node = 'fake_node'
  101. self.compute._get_resource_tracker(node).AndReturn(
  102. FakeResourceTracker())
  103. self.admin_context = False
  104. def fake_allocate(context, *args, **kwargs):
  105. if context.is_admin:
  106. self.admin_context = True
  107. # NOTE(vish): The nice mox parameter matchers here don't work well
  108. # because they raise an exception that gets wrapped by
  109. # the retry exception handling, so use a side effect
  110. # to keep track of whether allocate was called with admin
  111. # context.
  112. self.compute._allocate_network(mox.IgnoreArg(), instance,
  113. mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(),
  114. mox.IgnoreArg()).WithSideEffects(fake_allocate)
  115. self.mox.ReplayAll()
  116. instance, nw_info = self.compute._build_instance(self.context, {}, {},
  117. None, None, None, True,
  118. node, instance,
  119. {}, False)
  120. self.assertFalse(self.admin_context,
  121. "_allocate_network called with admin context")
  122. self.assertEqual(vm_states.BUILDING, instance.vm_state)
  123. self.assertEqual(task_states.BLOCK_DEVICE_MAPPING, instance.task_state)
  124. def test_reschedule_maintains_context(self):
  125. # override tracker with a version that causes a reschedule
  126. class FakeResourceTracker(object):
  127. def instance_claim(self, context, instance, limits):
  128. raise test.TestingException()
  129. self.mox.StubOutWithMock(self.compute, '_get_resource_tracker')
  130. self.mox.StubOutWithMock(self.compute, '_reschedule_or_error')
  131. self.mox.StubOutWithMock(objects.BlockDeviceMappingList,
  132. 'get_by_instance_uuid')
  133. instance = fake_instance.fake_instance_obj(self.context)
  134. objects.BlockDeviceMappingList.get_by_instance_uuid(
  135. mox.IgnoreArg(), instance.uuid).AndReturn([])
  136. node = 'fake_node'
  137. self.compute._get_resource_tracker(node).AndReturn(
  138. FakeResourceTracker())
  139. self.admin_context = False
  140. def fake_retry_or_error(context, *args, **kwargs):
  141. if context.is_admin:
  142. self.admin_context = True
  143. # NOTE(vish): we could use a mos parameter matcher here but it leads
  144. # to a very cryptic error message, so use the same method
  145. # as the allocate_network_maintains_context test.
  146. self.compute._reschedule_or_error(mox.IgnoreArg(), instance,
  147. mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(),
  148. mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(),
  149. mox.IgnoreArg(), mox.IgnoreArg(),
  150. mox.IgnoreArg()).WithSideEffects(fake_retry_or_error)
  151. self.mox.ReplayAll()
  152. self.assertRaises(test.TestingException,
  153. self.compute._build_instance, self.context, {}, {},
  154. None, None, None, True, node, instance, {}, False)
  155. self.assertFalse(self.admin_context,
  156. "_reschedule_or_error called with admin context")
  157. def test_allocate_network_fails(self):
  158. self.flags(network_allocate_retries=0)
  159. nwapi = self.compute.network_api
  160. self.mox.StubOutWithMock(nwapi, 'allocate_for_instance')
  161. instance = {}
  162. is_vpn = 'fake-is-vpn'
  163. req_networks = 'fake-req-networks'
  164. macs = 'fake-macs'
  165. sec_groups = 'fake-sec-groups'
  166. dhcp_options = None
  167. nwapi.allocate_for_instance(
  168. self.context, instance, vpn=is_vpn,
  169. requested_networks=req_networks, macs=macs,
  170. security_groups=sec_groups,
  171. dhcp_options=dhcp_options).AndRaise(test.TestingException())
  172. self.mox.ReplayAll()
  173. self.assertRaises(test.TestingException,
  174. self.compute._allocate_network_async,
  175. self.context, instance, req_networks, macs,
  176. sec_groups, is_vpn, dhcp_options)
  177. def test_allocate_network_neg_conf_value_treated_as_zero(self):
  178. self.flags(network_allocate_retries=-1)
  179. nwapi = self.compute.network_api
  180. self.mox.StubOutWithMock(nwapi, 'allocate_for_instance')
  181. instance = {}
  182. is_vpn = 'fake-is-vpn'
  183. req_networks = 'fake-req-networks'
  184. macs = 'fake-macs'
  185. sec_groups = 'fake-sec-groups'
  186. dhcp_options = None
  187. # Only attempted once.
  188. nwapi.allocate_for_instance(
  189. self.context, instance, vpn=is_vpn,
  190. requested_networks=req_networks, macs=macs,
  191. security_groups=sec_groups,
  192. dhcp_options=dhcp_options).AndRaise(test.TestingException())
  193. self.mox.ReplayAll()
  194. self.assertRaises(test.TestingException,
  195. self.compute._allocate_network_async,
  196. self.context, instance, req_networks, macs,
  197. sec_groups, is_vpn, dhcp_options)
  198. @mock.patch.object(network_api.API, 'allocate_for_instance')
  199. @mock.patch.object(manager.ComputeManager, '_instance_update')
  200. @mock.patch.object(time, 'sleep')
  201. def test_allocate_network_with_conf_value_is_one(
  202. self, sleep, _instance_update, allocate_for_instance):
  203. self.flags(network_allocate_retries=1)
  204. instance = fake_instance.fake_instance_obj(
  205. self.context, expected_attrs=['system_metadata'])
  206. is_vpn = 'fake-is-vpn'
  207. req_networks = 'fake-req-networks'
  208. macs = 'fake-macs'
  209. sec_groups = 'fake-sec-groups'
  210. dhcp_options = None
  211. final_result = 'zhangtralon'
  212. allocate_for_instance.side_effect = [test.TestingException(),
  213. final_result]
  214. res = self.compute._allocate_network_async(self.context, instance,
  215. req_networks,
  216. macs,
  217. sec_groups,
  218. is_vpn,
  219. dhcp_options)
  220. self.assertEqual(final_result, res)
  221. self.assertEqual(1, sleep.call_count)
  222. def test_init_host(self):
  223. our_host = self.compute.host
  224. fake_context = 'fake-context'
  225. inst = fake_instance.fake_db_instance(
  226. vm_state=vm_states.ACTIVE,
  227. info_cache=dict(test_instance_info_cache.fake_info_cache,
  228. network_info=None),
  229. security_groups=None)
  230. startup_instances = [inst, inst, inst]
  231. def _do_mock_calls(defer_iptables_apply):
  232. self.compute.driver.init_host(host=our_host)
  233. context.get_admin_context().AndReturn(fake_context)
  234. db.instance_get_all_by_host(
  235. fake_context, our_host, columns_to_join=['info_cache'],
  236. use_slave=False
  237. ).AndReturn(startup_instances)
  238. if defer_iptables_apply:
  239. self.compute.driver.filter_defer_apply_on()
  240. self.compute._destroy_evacuated_instances(fake_context)
  241. self.compute._init_instance(fake_context,
  242. mox.IsA(objects.Instance))
  243. self.compute._init_instance(fake_context,
  244. mox.IsA(objects.Instance))
  245. self.compute._init_instance(fake_context,
  246. mox.IsA(objects.Instance))
  247. if defer_iptables_apply:
  248. self.compute.driver.filter_defer_apply_off()
  249. self.mox.StubOutWithMock(self.compute.driver, 'init_host')
  250. self.mox.StubOutWithMock(self.compute.driver,
  251. 'filter_defer_apply_on')
  252. self.mox.StubOutWithMock(self.compute.driver,
  253. 'filter_defer_apply_off')
  254. self.mox.StubOutWithMock(db, 'instance_get_all_by_host')
  255. self.mox.StubOutWithMock(context, 'get_admin_context')
  256. self.mox.StubOutWithMock(self.compute,
  257. '_destroy_evacuated_instances')
  258. self.mox.StubOutWithMock(self.compute,
  259. '_init_instance')
  260. # Test with defer_iptables_apply
  261. self.flags(defer_iptables_apply=True)
  262. _do_mock_calls(True)
  263. self.mox.ReplayAll()
  264. self.compute.init_host()
  265. self.mox.VerifyAll()
  266. # Test without defer_iptables_apply
  267. self.mox.ResetAll()
  268. self.flags(defer_iptables_apply=False)
  269. _do_mock_calls(False)
  270. self.mox.ReplayAll()
  271. self.compute.init_host()
  272. # tearDown() uses context.get_admin_context(), so we have
  273. # to do the verification here and unstub it.
  274. self.mox.VerifyAll()
  275. self.mox.UnsetStubs()
  276. @mock.patch('nova.objects.InstanceList')
  277. def test_cleanup_host(self, mock_instance_list):
  278. # just testing whether the cleanup_host method
  279. # when fired will invoke the underlying driver's
  280. # equivalent method.
  281. mock_instance_list.get_by_host.return_value = []
  282. with mock.patch.object(self.compute, 'driver') as mock_driver:
  283. self.compute.init_host()
  284. mock_driver.init_host.assert_called_once_with(host='fake-mini')
  285. self.compute.cleanup_host()
  286. mock_driver.cleanup_host.assert_called_once_with(host='fake-mini')
  287. def test_init_host_with_deleted_migration(self):
  288. our_host = self.compute.host
  289. not_our_host = 'not-' + our_host
  290. fake_context = 'fake-context'
  291. deleted_instance = fake_instance.fake_instance_obj(
  292. self.context, host=not_our_host, uuid='fake-uuid')
  293. self.mox.StubOutWithMock(self.compute.driver, 'init_host')
  294. self.mox.StubOutWithMock(self.compute.driver, 'destroy')
  295. self.mox.StubOutWithMock(db, 'instance_get_all_by_host')
  296. self.mox.StubOutWithMock(context, 'get_admin_context')
  297. self.mox.StubOutWithMock(self.compute, 'init_virt_events')
  298. self.mox.StubOutWithMock(self.compute, '_get_instances_on_driver')
  299. self.mox.StubOutWithMock(self.compute, '_init_instance')
  300. self.mox.StubOutWithMock(self.compute, '_get_instance_nw_info')
  301. self.compute.driver.init_host(host=our_host)
  302. context.get_admin_context().AndReturn(fake_context)
  303. db.instance_get_all_by_host(fake_context, our_host,
  304. columns_to_join=['info_cache'],
  305. use_slave=False
  306. ).AndReturn([])
  307. self.compute.init_virt_events()
  308. # simulate failed instance
  309. self.compute._get_instances_on_driver(
  310. fake_context, {'deleted': False}).AndReturn([deleted_instance])
  311. self.compute._get_instance_nw_info(fake_context, deleted_instance
  312. ).AndRaise(exception.InstanceNotFound(
  313. instance_id=deleted_instance['uuid']))
  314. # ensure driver.destroy is called so that driver may
  315. # clean up any dangling files
  316. self.compute.driver.destroy(fake_context, deleted_instance,
  317. mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg())
  318. self.mox.ReplayAll()
  319. self.compute.init_host()
  320. # tearDown() uses context.get_admin_context(), so we have
  321. # to do the verification here and unstub it.
  322. self.mox.VerifyAll()
  323. self.mox.UnsetStubs()
  324. def test_init_instance_failed_resume_sets_error(self):
  325. instance = fake_instance.fake_instance_obj(
  326. self.context,
  327. uuid='fake-uuid',
  328. info_cache=None,
  329. power_state=power_state.RUNNING,
  330. vm_state=vm_states.ACTIVE,
  331. task_state=None,
  332. expected_attrs=['info_cache'])
  333. self.flags(resume_guests_state_on_host_boot=True)
  334. self.mox.StubOutWithMock(self.compute, '_get_power_state')
  335. self.mox.StubOutWithMock(self.compute.driver, 'plug_vifs')
  336. self.mox.StubOutWithMock(self.compute.driver,
  337. 'resume_state_on_host_boot')
  338. self.mox.StubOutWithMock(self.compute,
  339. '_get_instance_block_device_info')
  340. self.mox.StubOutWithMock(self.compute,
  341. '_set_instance_error_state')
  342. self.compute._get_power_state(mox.IgnoreArg(),
  343. instance).AndReturn(power_state.SHUTDOWN)
  344. self.compute._get_power_state(mox.IgnoreArg(),
  345. instance).AndReturn(power_state.SHUTDOWN)
  346. self.compute._get_power_state(mox.IgnoreArg(),
  347. instance).AndReturn(power_state.SHUTDOWN)
  348. self.compute.driver.plug_vifs(instance, mox.IgnoreArg())
  349. self.compute._get_instance_block_device_info(mox.IgnoreArg(),
  350. instance).AndReturn('fake-bdm')
  351. self.compute.driver.resume_state_on_host_boot(mox.IgnoreArg(),
  352. instance, mox.IgnoreArg(),
  353. 'fake-bdm').AndRaise(test.TestingException)
  354. self.compute._set_instance_error_state(mox.IgnoreArg(), instance)
  355. self.mox.ReplayAll()
  356. self.compute._init_instance('fake-context', instance)
  357. def test_init_instance_stuck_in_deleting(self):
  358. instance = fake_instance.fake_instance_obj(
  359. self.context,
  360. uuid='fake-uuid',
  361. power_state=power_state.RUNNING,
  362. vm_state=vm_states.ACTIVE,
  363. task_state=task_states.DELETING)
  364. self.mox.StubOutWithMock(objects.BlockDeviceMappingList,
  365. 'get_by_instance_uuid')
  366. self.mox.StubOutWithMock(self.compute, '_delete_instance')
  367. self.mox.StubOutWithMock(instance, 'obj_load_attr')
  368. bdms = []
  369. instance.obj_load_attr('metadata')
  370. instance.obj_load_attr('system_metadata')
  371. objects.BlockDeviceMappingList.get_by_instance_uuid(
  372. self.context, instance.uuid).AndReturn(bdms)
  373. self.compute._delete_instance(self.context, instance, bdms,
  374. mox.IgnoreArg())
  375. self.mox.ReplayAll()
  376. self.compute._init_instance(self.context, instance)
  377. def _test_init_instance_reverts_crashed_migrations(self,
  378. old_vm_state=None):
  379. power_on = True if (not old_vm_state or
  380. old_vm_state == vm_states.ACTIVE) else False
  381. sys_meta = {
  382. 'old_vm_state': old_vm_state
  383. }
  384. instance = fake_instance.fake_instance_obj(
  385. self.context,
  386. uuid='foo',
  387. vm_state=vm_states.ERROR,
  388. task_state=task_states.RESIZE_MIGRATING,
  389. power_state=power_state.SHUTDOWN,
  390. system_metadata=sys_meta,
  391. expected_attrs=['system_metadata'])
  392. self.mox.StubOutWithMock(compute_utils, 'get_nw_info_for_instance')
  393. self.mox.StubOutWithMock(self.compute.driver, 'plug_vifs')
  394. self.mox.StubOutWithMock(self.compute.driver,
  395. 'finish_revert_migration')
  396. self.mox.StubOutWithMock(self.compute,
  397. '_get_instance_block_device_info')
  398. self.mox.StubOutWithMock(self.compute.driver, 'get_info')
  399. self.mox.StubOutWithMock(instance, 'save')
  400. self.mox.StubOutWithMock(self.compute, '_retry_reboot')
  401. self.compute._retry_reboot(self.context, instance).AndReturn(
  402. (False, None))
  403. compute_utils.get_nw_info_for_instance(instance).AndReturn(
  404. network_model.NetworkInfo())
  405. self.compute.driver.plug_vifs(instance, [])
  406. self.compute._get_instance_block_device_info(
  407. self.context, instance).AndReturn([])
  408. self.compute.driver.finish_revert_migration(self.context, instance,
  409. [], [], power_on)
  410. instance.save()
  411. self.compute.driver.get_info(instance).AndReturn(
  412. {'state': power_state.SHUTDOWN})
  413. self.compute.driver.get_info(instance).AndReturn(
  414. {'state': power_state.SHUTDOWN})
  415. self.mox.ReplayAll()
  416. self.compute._init_instance(self.context, instance)
  417. self.assertIsNone(instance.task_state)
  418. def test_init_instance_reverts_crashed_migration_from_active(self):
  419. self._test_init_instance_reverts_crashed_migrations(
  420. old_vm_state=vm_states.ACTIVE)
  421. def test_init_instance_reverts_crashed_migration_from_stopped(self):
  422. self._test_init_instance_reverts_crashed_migrations(
  423. old_vm_state=vm_states.STOPPED)
  424. def test_init_instance_reverts_crashed_migration_no_old_state(self):
  425. self._test_init_instance_reverts_crashed_migrations(old_vm_state=None)
  426. def test_init_instance_resets_crashed_live_migration(self):
  427. instance = fake_instance.fake_instance_obj(
  428. self.context,
  429. uuid='foo',
  430. vm_state=vm_states.ACTIVE,
  431. task_state=task_states.MIGRATING)
  432. with contextlib.nested(
  433. mock.patch.object(instance, 'save'),
  434. mock.patch('nova.compute.utils.get_nw_info_for_instance',
  435. return_value=network_model.NetworkInfo())
  436. ) as (save, get_nw_info):
  437. self.compute._init_instance(self.context, instance)
  438. save.assert_called_once_with(expected_task_state=['migrating'])
  439. get_nw_info.assert_called_once_with(instance)
  440. self.assertIsNone(instance.task_state)
  441. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  442. def _test_init_instance_sets_building_error(self, vm_state,
  443. task_state=None):
  444. instance = fake_instance.fake_instance_obj(
  445. self.context,
  446. uuid='foo',
  447. vm_state=vm_state,
  448. task_state=task_state)
  449. with mock.patch.object(instance, 'save') as save:
  450. self.compute._init_instance(self.context, instance)
  451. save.assert_called_once_with()
  452. self.assertIsNone(instance.task_state)
  453. self.assertEqual(vm_states.ERROR, instance.vm_state)
  454. def test_init_instance_sets_building_error(self):
  455. self._test_init_instance_sets_building_error(vm_states.BUILDING)
  456. def test_init_instance_sets_rebuilding_errors(self):
  457. tasks = [task_states.REBUILDING,
  458. task_states.REBUILD_BLOCK_DEVICE_MAPPING,
  459. task_states.REBUILD_SPAWNING]
  460. vms = [vm_states.ACTIVE, vm_states.STOPPED]
  461. for vm_state in vms:
  462. for task_state in tasks:
  463. self._test_init_instance_sets_building_error(
  464. vm_state, task_state)
  465. def _test_init_instance_sets_building_tasks_error(self, instance):
  466. with mock.patch.object(instance, 'save') as save:
  467. self.compute._init_instance(self.context, instance)
  468. save.assert_called_once_with()
  469. self.assertIsNone(instance.task_state)
  470. self.assertEqual(vm_states.ERROR, instance.vm_state)
  471. def test_init_instance_sets_building_tasks_error_scheduling(self):
  472. instance = fake_instance.fake_instance_obj(
  473. self.context,
  474. uuid='foo',
  475. vm_state=None,
  476. task_state=task_states.SCHEDULING)
  477. self._test_init_instance_sets_building_tasks_error(instance)
  478. def test_init_instance_sets_building_tasks_error_block_device(self):
  479. instance = objects.Instance(self.context)
  480. instance.uuid = 'foo'
  481. instance.vm_state = None
  482. instance.task_state = task_states.BLOCK_DEVICE_MAPPING
  483. self._test_init_instance_sets_building_tasks_error(instance)
  484. def test_init_instance_sets_building_tasks_error_networking(self):
  485. instance = objects.Instance(self.context)
  486. instance.uuid = 'foo'
  487. instance.vm_state = None
  488. instance.task_state = task_states.NETWORKING
  489. self._test_init_instance_sets_building_tasks_error(instance)
  490. def test_init_instance_sets_building_tasks_error_spawning(self):
  491. instance = objects.Instance(self.context)
  492. instance.uuid = 'foo'
  493. instance.vm_state = None
  494. instance.task_state = task_states.SPAWNING
  495. self._test_init_instance_sets_building_tasks_error(instance)
  496. def _test_init_instance_cleans_image_states(self, instance):
  497. with mock.patch.object(instance, 'save') as save:
  498. self.compute._get_power_state = mock.Mock()
  499. self.compute.driver.post_interrupted_snapshot_cleanup = mock.Mock()
  500. instance.info_cache = None
  501. instance.power_state = power_state.RUNNING
  502. self.compute._init_instance(self.context, instance)
  503. save.assert_called_once_with()
  504. self.compute.driver.post_interrupted_snapshot_cleanup.\
  505. assert_called_once_with(self.context, instance)
  506. self.assertIsNone(instance.task_state)
  507. def test_init_instance_cleans_image_state_pending_upload(self):
  508. instance = objects.Instance(self.context)
  509. instance.uuid = 'foo'
  510. instance.vm_state = vm_states.ACTIVE
  511. instance.task_state = task_states.IMAGE_PENDING_UPLOAD
  512. self._test_init_instance_cleans_image_states(instance)
  513. def test_init_instance_cleans_image_state_uploading(self):
  514. instance = objects.Instance(self.context)
  515. instance.uuid = 'foo'
  516. instance.vm_state = vm_states.ACTIVE
  517. instance.task_state = task_states.IMAGE_UPLOADING
  518. self._test_init_instance_cleans_image_states(instance)
  519. def test_init_instance_cleans_image_state_snapshot(self):
  520. instance = objects.Instance(self.context)
  521. instance.uuid = 'foo'
  522. instance.vm_state = vm_states.ACTIVE
  523. instance.task_state = task_states.IMAGE_SNAPSHOT
  524. self._test_init_instance_cleans_image_states(instance)
  525. def test_init_instance_cleans_image_state_snapshot_pending(self):
  526. instance = objects.Instance(self.context)
  527. instance.uuid = 'foo'
  528. instance.vm_state = vm_states.ACTIVE
  529. instance.task_state = task_states.IMAGE_SNAPSHOT_PENDING
  530. self._test_init_instance_cleans_image_states(instance)
  531. def test_init_instance_errors_when_not_migrating(self):
  532. instance = objects.Instance(self.context)
  533. instance.uuid = 'foo'
  534. instance.vm_state = vm_states.ERROR
  535. instance.task_state = task_states.IMAGE_UPLOADING
  536. self.mox.StubOutWithMock(compute_utils, 'get_nw_info_for_instance')
  537. self.mox.ReplayAll()
  538. self.compute._init_instance(self.context, instance)
  539. self.mox.VerifyAll()
  540. def test_init_instance_deletes_error_deleting_instance(self):
  541. instance = fake_instance.fake_instance_obj(
  542. self.context,
  543. uuid='fake',
  544. vm_state=vm_states.ERROR,
  545. task_state=task_states.DELETING)
  546. self.mox.StubOutWithMock(objects.BlockDeviceMappingList,
  547. 'get_by_instance_uuid')
  548. self.mox.StubOutWithMock(self.compute, '_delete_instance')
  549. self.mox.StubOutWithMock(instance, 'obj_load_attr')
  550. bdms = []
  551. instance.obj_load_attr('metadata')
  552. instance.obj_load_attr('system_metadata')
  553. objects.BlockDeviceMappingList.get_by_instance_uuid(
  554. self.context, instance.uuid).AndReturn(bdms)
  555. self.compute._delete_instance(self.context, instance, bdms,
  556. mox.IgnoreArg())
  557. self.mox.ReplayAll()
  558. self.compute._init_instance(self.context, instance)
  559. self.mox.VerifyAll()
  560. @mock.patch('nova.context.RequestContext.elevated')
  561. @mock.patch('nova.compute.utils.get_nw_info_for_instance')
  562. @mock.patch(
  563. 'nova.compute.manager.ComputeManager._get_instance_block_device_info')
  564. @mock.patch('nova.virt.driver.ComputeDriver.destroy')
  565. @mock.patch('nova.virt.driver.ComputeDriver.get_volume_connector')
  566. def test_shutdown_instance_endpoint_not_found(self, mock_connector,
  567. mock_destroy, mock_blk_device_info, mock_nw_info, mock_elevated):
  568. mock_connector.side_effect = cinder_exception.EndpointNotFound
  569. mock_elevated.return_value = self.context
  570. instance = fake_instance.fake_instance_obj(
  571. self.context,
  572. uuid='fake',
  573. vm_state=vm_states.ERROR,
  574. task_state=task_states.DELETING)
  575. bdms = [mock.Mock(id=1, is_volume=True)]
  576. self.compute._shutdown_instance(self.context, instance, bdms,
  577. notify=False, try_deallocate_networks=False)
  578. def _test_init_instance_retries_reboot(self, instance, reboot_type,
  579. return_power_state):
  580. with contextlib.nested(
  581. mock.patch.object(self.compute, '_get_power_state',
  582. return_value=return_power_state),
  583. mock.patch.object(self.compute.compute_rpcapi, 'reboot_instance'),
  584. mock.patch.object(compute_utils, 'get_nw_info_for_instance')
  585. ) as (
  586. _get_power_state,
  587. reboot_instance,
  588. get_nw_info_for_instance
  589. ):
  590. self.compute._init_instance(self.context, instance)
  591. call = mock.call(self.context, instance, block_device_info=None,
  592. reboot_type=reboot_type)
  593. reboot_instance.assert_has_calls([call])
  594. def test_init_instance_retries_reboot_pending(self):
  595. instance = objects.Instance(self.context)
  596. instance.uuid = 'foo'
  597. instance.task_state = task_states.REBOOT_PENDING
  598. for state in vm_states.ALLOW_SOFT_REBOOT:
  599. instance.vm_state = state
  600. self._test_init_instance_retries_reboot(instance, 'SOFT',
  601. power_state.RUNNING)
  602. def test_init_instance_retries_reboot_pending_hard(self):
  603. instance = objects.Instance(self.context)
  604. instance.uuid = 'foo'
  605. instance.task_state = task_states.REBOOT_PENDING_HARD
  606. for state in vm_states.ALLOW_HARD_REBOOT:
  607. # NOTE(dave-mcnally) while a reboot of a vm in error state is
  608. # possible we don't attempt to recover an error during init
  609. if state == vm_states.ERROR:
  610. continue
  611. instance.vm_state = state
  612. self._test_init_instance_retries_reboot(instance, 'HARD',
  613. power_state.RUNNING)
  614. def test_init_instance_retries_reboot_started(self):
  615. instance = objects.Instance(self.context)
  616. instance.uuid = 'foo'
  617. instance.vm_state = vm_states.ACTIVE
  618. instance.task_state = task_states.REBOOT_STARTED
  619. self._test_init_instance_retries_reboot(instance, 'HARD',
  620. power_state.NOSTATE)
  621. def test_init_instance_retries_reboot_started_hard(self):
  622. instance = objects.Instance(self.context)
  623. instance.uuid = 'foo'
  624. instance.vm_state = vm_states.ACTIVE
  625. instance.task_state = task_states.REBOOT_STARTED_HARD
  626. self._test_init_instance_retries_reboot(instance, 'HARD',
  627. power_state.NOSTATE)
  628. def _test_init_instance_cleans_reboot_state(self, instance):
  629. with contextlib.nested(
  630. mock.patch.object(self.compute, '_get_power_state',
  631. return_value=power_state.RUNNING),
  632. mock.patch.object(instance, 'save', autospec=True),
  633. mock.patch.object(compute_utils, 'get_nw_info_for_instance')
  634. ) as (
  635. _get_power_state,
  636. instance_save,
  637. get_nw_info_for_instance
  638. ):
  639. self.compute._init_instance(self.context, instance)
  640. instance_save.assert_called_once_with()
  641. self.assertIsNone(instance.task_state)
  642. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  643. def test_init_instance_cleans_image_state_reboot_started(self):
  644. instance = objects.Instance(self.context)
  645. instance.uuid = 'foo'
  646. instance.vm_state = vm_states.ACTIVE
  647. instance.task_state = task_states.REBOOT_STARTED
  648. instance.power_state = power_state.RUNNING
  649. self._test_init_instance_cleans_reboot_state(instance)
  650. def test_init_instance_cleans_image_state_reboot_started_hard(self):
  651. instance = objects.Instance(self.context)
  652. instance.uuid = 'foo'
  653. instance.vm_state = vm_states.ACTIVE
  654. instance.task_state = task_states.REBOOT_STARTED_HARD
  655. instance.power_state = power_state.RUNNING
  656. self._test_init_instance_cleans_reboot_state(instance)
  657. def test_init_instance_retries_power_off(self):
  658. instance = objects.Instance(self.context)
  659. instance.uuid = 'foo'
  660. instance.id = 1
  661. instance.vm_state = vm_states.ACTIVE
  662. instance.task_state = task_states.POWERING_OFF
  663. with mock.patch.object(self.compute, 'stop_instance'):
  664. self.compute._init_instance(self.context, instance)
  665. call = mock.call(self.context, instance)
  666. self.compute.stop_instance.assert_has_calls([call])
  667. def test_init_instance_retries_power_on(self):
  668. instance = objects.Instance(self.context)
  669. instance.uuid = 'foo'
  670. instance.id = 1
  671. instance.vm_state = vm_states.ACTIVE
  672. instance.task_state = task_states.POWERING_ON
  673. with mock.patch.object(self.compute, 'start_instance'):
  674. self.compute._init_instance(self.context, instance)
  675. call = mock.call(self.context, instance)
  676. self.compute.start_instance.assert_has_calls([call])
  677. def test_init_instance_retries_power_on_silent_exception(self):
  678. instance = objects.Instance(self.context)
  679. instance.uuid = 'foo'
  680. instance.id = 1
  681. instance.vm_state = vm_states.ACTIVE
  682. instance.task_state = task_states.POWERING_ON
  683. with mock.patch.object(self.compute, 'start_instance',
  684. return_value=Exception):
  685. init_return = self.compute._init_instance(self.context, instance)
  686. call = mock.call(self.context, instance)
  687. self.compute.start_instance.assert_has_calls([call])
  688. self.assertIsNone(init_return)
  689. def test_init_instance_retries_power_off_silent_exception(self):
  690. instance = objects.Instance(self.context)
  691. instance.uuid = 'foo'
  692. instance.id = 1
  693. instance.vm_state = vm_states.ACTIVE
  694. instance.task_state = task_states.POWERING_OFF
  695. with mock.patch.object(self.compute, 'stop_instance',
  696. return_value=Exception):
  697. init_return = self.compute._init_instance(self.context, instance)
  698. call = mock.call(self.context, instance)
  699. self.compute.stop_instance.assert_has_calls([call])
  700. self.assertIsNone(init_return)
  701. def test_get_instances_on_driver(self):
  702. fake_context = context.get_admin_context()
  703. driver_instances = []
  704. for x in xrange(10):
  705. driver_instances.append(fake_instance.fake_db_instance())
  706. self.mox.StubOutWithMock(self.compute.driver,
  707. 'list_instance_uuids')
  708. self.mox.StubOutWithMock(db, 'instance_get_all_by_filters')
  709. self.compute.driver.list_instance_uuids().AndReturn(
  710. [inst['uuid'] for inst in driver_instances])
  711. db.instance_get_all_by_filters(
  712. fake_context,
  713. {'uuid': [inst['uuid'] for
  714. inst in driver_instances]},
  715. 'created_at', 'desc', columns_to_join=None,
  716. limit=None, marker=None,
  717. use_slave=True).AndReturn(
  718. driver_instances)
  719. self.mox.ReplayAll()
  720. result = self.compute._get_instances_on_driver(fake_context)
  721. self.assertEqual([x['uuid'] for x in driver_instances],
  722. [x['uuid'] for x in result])
  723. @mock.patch('nova.virt.driver.ComputeDriver.list_instance_uuids')
  724. @mock.patch('nova.db.api.instance_get_all_by_filters')
  725. def test_get_instances_on_driver_empty(self, mock_list, mock_db):
  726. fake_context = context.get_admin_context()
  727. mock_list.return_value = []
  728. result = self.compute._get_instances_on_driver(fake_context)
  729. # instance_get_all_by_filters should not be called
  730. self.assertEqual(0, mock_db.call_count)
  731. self.assertEqual([],
  732. [x['uuid'] for x in result])
  733. def test_get_instances_on_driver_fallback(self):
  734. # Test getting instances when driver doesn't support
  735. # 'list_instance_uuids'
  736. self.compute.host = 'host'
  737. filters = {'host': self.compute.host}
  738. fake_context = context.get_admin_context()
  739. self.flags(instance_name_template='inst-%i')
  740. all_instances = []
  741. driver_instances = []
  742. for x in xrange(10):
  743. instance = fake_instance.fake_db_instance(name='inst-%i' % x,
  744. id=x)
  745. if x % 2:
  746. driver_instances.append(instance)
  747. all_instances.append(instance)
  748. self.mox.StubOutWithMock(self.compute.driver,
  749. 'list_instance_uuids')
  750. self.mox.StubOutWithMock(self.compute.driver,
  751. 'list_instances')
  752. self.mox.StubOutWithMock(db, 'instance_get_all_by_filters')
  753. self.compute.driver.list_instance_uuids().AndRaise(
  754. NotImplementedError())
  755. self.compute.driver.list_instances().AndReturn(
  756. [inst['name'] for inst in driver_instances])
  757. db.instance_get_all_by_filters(
  758. fake_context, filters,
  759. 'created_at', 'desc', columns_to_join=None,
  760. limit=None, marker=None,
  761. use_slave=True).AndReturn(all_instances)
  762. self.mox.ReplayAll()
  763. result = self.compute._get_instances_on_driver(fake_context, filters)
  764. self.assertEqual([x['uuid'] for x in driver_instances],
  765. [x['uuid'] for x in result])
  766. def test_instance_usage_audit(self):
  767. instances = [objects.Instance(uuid='foo')]
  768. @classmethod
  769. def fake_get(*a, **k):
  770. return instances
  771. self.flags(instance_usage_audit=True)
  772. self.stubs.Set(compute_utils, 'has_audit_been_run',
  773. lambda *a, **k: False)
  774. self.stubs.Set(objects.InstanceList,
  775. 'get_active_by_window_joined', fake_get)
  776. self.stubs.Set(compute_utils, 'start_instance_usage_audit',
  777. lambda *a, **k: None)
  778. self.stubs.Set(compute_utils, 'finish_instance_usage_audit',
  779. lambda *a, **k: None)
  780. self.mox.StubOutWithMock(self.compute.conductor_api,
  781. 'notify_usage_exists')
  782. self.compute.conductor_api.notify_usage_exists(
  783. self.context, instances[0], ignore_missing_network_data=False)
  784. self.mox.ReplayAll()
  785. self.compute._instance_usage_audit(self.context)
  786. def _get_sync_instance(self, power_state, vm_state, task_state=None,
  787. shutdown_terminate=False):
  788. instance = objects.Instance()
  789. instance.uuid = 'fake-uuid'
  790. instance.power_state = power_state
  791. instance.vm_state = vm_state
  792. instance.host = self.compute.host
  793. instance.task_state = task_state
  794. instance.shutdown_terminate = shutdown_terminate
  795. self.mox.StubOutWithMock(instance, 'refresh')
  796. self.mox.StubOutWithMock(instance, 'save')
  797. return instance
  798. def test_sync_instance_power_state_match(self):
  799. instance = self._get_sync_instance(power_state.RUNNING,
  800. vm_states.ACTIVE)
  801. instance.refresh(use_slave=False)
  802. self.mox.ReplayAll()
  803. self.compute._sync_instance_power_state(self.context, instance,
  804. power_state.RUNNING)
  805. def test_sync_instance_power_state_running_stopped(self):
  806. instance = self._get_sync_instance(power_state.RUNNING,
  807. vm_states.ACTIVE)
  808. instance.refresh(use_slave=False)
  809. instance.save()
  810. self.mox.ReplayAll()
  811. self.compute._sync_instance_power_state(self.context, instance,
  812. power_state.SHUTDOWN)
  813. self.assertEqual(instance.power_state, power_state.SHUTDOWN)
  814. def _test_sync_to_stop(self, power_state, vm_state, driver_power_state,
  815. stop=True, force=False, shutdown_terminate=False):
  816. instance = self._get_sync_instance(
  817. power_state, vm_state, shutdown_terminate=shutdown_terminate)
  818. instance.refresh(use_slave=False)
  819. instance.save()
  820. self.mox.StubOutWithMock(self.compute.compute_api, 'stop')
  821. self.mox.StubOutWithMock(self.compute.compute_api, 'delete')
  822. self.mox.StubOutWithMock(self.compute.compute_api, 'force_stop')
  823. if shutdown_terminate:
  824. self.compute.compute_api.delete(self.context, instance)
  825. elif stop:
  826. if force:
  827. self.compute.compute_api.force_stop(self.context, instance)
  828. else:
  829. self.compute.compute_api.stop(self.context, instance)
  830. self.mox.ReplayAll()
  831. self.compute._sync_instance_power_state(self.context, instance,
  832. driver_power_state)
  833. self.mox.VerifyAll()
  834. self.mox.UnsetStubs()
  835. def test_sync_instance_power_state_to_stop(self):
  836. for ps in (power_state.SHUTDOWN, power_state.CRASHED,
  837. power_state.SUSPENDED):
  838. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE, ps)
  839. for ps in (power_state.SHUTDOWN, power_state.CRASHED):
  840. self._test_sync_to_stop(power_state.PAUSED, vm_states.PAUSED, ps,
  841. force=True)
  842. self._test_sync_to_stop(power_state.SHUTDOWN, vm_states.STOPPED,
  843. power_state.RUNNING, force=True)
  844. def test_sync_instance_power_state_to_terminate(self):
  845. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE,
  846. power_state.SHUTDOWN,
  847. force=False, shutdown_terminate=True)
  848. def test_sync_instance_power_state_to_no_stop(self):
  849. for ps in (power_state.PAUSED, power_state.NOSTATE):
  850. self._test_sync_to_stop(power_state.RUNNING, vm_states.ACTIVE, ps,
  851. stop=False)
  852. for vs in (vm_states.SOFT_DELETED, vm_states.DELETED):
  853. for ps in (power_state.NOSTATE, power_state.SHUTDOWN):
  854. self._test_sync_to_stop(power_state.RUNNING, vs, ps,
  855. stop=False)
  856. @mock.patch('nova.compute.manager.ComputeManager.'
  857. '_sync_instance_power_state')
  858. def test_query_driver_power_state_and_sync_pending_task(
  859. self, mock_sync_power_state):
  860. with mock.patch.object(self.compute.driver,
  861. 'get_info') as mock_get_info:
  862. db_instance = objects.Instance(uuid='fake-uuid',
  863. task_state=task_states.POWERING_OFF)
  864. self.compute._query_driver_power_state_and_sync(self.context,
  865. db_instance)
  866. self.assertFalse(mock_get_info.called)
  867. self.assertFalse(mock_sync_power_state.called)
  868. @mock.patch('nova.compute.manager.ComputeManager.'
  869. '_sync_instance_power_state')
  870. def test_query_driver_power_state_and_sync_not_found_driver(
  871. self, mock_sync_power_state):
  872. error = exception.InstanceNotFound(instance_id=1)
  873. with mock.patch.object(self.compute.driver,
  874. 'get_info', side_effect=error) as mock_get_info:
  875. db_instance = objects.Instance(uuid='fake-uuid', task_state=None)
  876. self.compute._query_driver_power_state_and_sync(self.context,
  877. db_instance)
  878. mock_get_info.assert_called_once_with(db_instance)
  879. mock_sync_power_state.assert_called_once_with(self.context,
  880. db_instance,
  881. power_state.NOSTATE,
  882. use_slave=True)
  883. def test_run_pending_deletes(self):
  884. self.flags(instance_delete_interval=10)
  885. class FakeInstance(object):
  886. def __init__(self, uuid, name, smd):
  887. self.uuid = uuid
  888. self.name = name
  889. self.system_metadata = smd
  890. self.cleaned = False
  891. def __getitem__(self, name):
  892. return getattr(self, name)
  893. def save(self, context):
  894. pass
  895. class FakeInstanceList(object):
  896. def get_by_filters(self, *args, **kwargs):
  897. return []
  898. a = FakeInstance('123', 'apple', {'clean_attempts': '100'})
  899. b = FakeInstance('456', 'orange', {'clean_attempts': '3'})
  900. c = FakeInstance('789', 'banana', {})
  901. self.mox.StubOutWithMock(objects.InstanceList,
  902. 'get_by_filters')
  903. objects.InstanceList.get_by_filters(
  904. {'read_deleted': 'yes'},
  905. {'deleted': True, 'soft_deleted': False, 'host': 'fake-mini',
  906. 'cleaned': False},
  907. expected_attrs=['info_cache', 'security_groups',
  908. 'system_metadata'],
  909. use_slave=True).AndReturn([a, b, c])
  910. self.mox.StubOutWithMock(self.compute.driver, 'delete_instance_files')
  911. self.compute.driver.delete_instance_files(
  912. mox.IgnoreArg()).AndReturn(True)
  913. self.compute.driver.delete_instance_files(
  914. mox.IgnoreArg()).AndReturn(False)
  915. self.mox.ReplayAll()
  916. self.compute._run_pending_deletes({})
  917. self.assertFalse(a.cleaned)
  918. self.assertEqual('100', a.system_metadata['clean_attempts'])
  919. self.assertTrue(b.cleaned)
  920. self.assertEqual('4', b.system_metadata['clean_attempts'])
  921. self.assertFalse(c.cleaned)
  922. self.assertEqual('1', c.system_metadata['clean_attempts'])
  923. def test_attach_interface_failure(self):
  924. # Test that the fault methods are invoked when an attach fails
  925. db_instance = fake_instance.fake_db_instance()
  926. f_instance = objects.Instance._from_db_object(self.context,
  927. objects.Instance(),
  928. db_instance)
  929. e = exception.InterfaceAttachFailed(instance_uuid=f_instance.uuid)
  930. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  931. @mock.patch.object(self.compute.network_api,
  932. 'allocate_port_for_instance',
  933. side_effect=e)
  934. def do_test(meth, add_fault):
  935. self.assertRaises(exception.InterfaceAttachFailed,
  936. self.compute.attach_interface,
  937. self.context, f_instance, 'net_id', 'port_id',
  938. None)
  939. add_fault.assert_has_calls(
  940. mock.call(self.context, f_instance, e,
  941. mock.ANY))
  942. do_test()
  943. def test_detach_interface_failure(self):
  944. # Test that the fault methods are invoked when a detach fails
  945. # Build test data that will cause a PortNotFound exception
  946. f_instance = mock.MagicMock()
  947. f_instance.info_cache = mock.MagicMock()
  948. f_instance.info_cache.network_info = []
  949. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  950. @mock.patch.object(self.compute, '_set_instance_error_state')
  951. def do_test(meth, add_fault):
  952. self.assertRaises(exception.PortNotFound,
  953. self.compute.detach_interface,
  954. self.context, f_instance, 'port_id')
  955. add_fault.assert_has_calls(
  956. mock.call(self.context, f_instance, mock.ANY, mock.ANY))
  957. do_test()
  958. def test_swap_volume_volume_api_usage(self):
  959. # This test ensures that volume_id arguments are passed to volume_api
  960. # and that volume states are OK
  961. volumes = {}
  962. old_volume_id = uuidutils.generate_uuid()
  963. volumes[old_volume_id] = {'id': old_volume_id,
  964. 'display_name': 'old_volume',
  965. 'status': 'detaching',
  966. 'size': 1}
  967. new_volume_id = uuidutils.generate_uuid()
  968. volumes[new_volume_id] = {'id': new_volume_id,
  969. 'display_name': 'new_volume',
  970. 'status': 'available',
  971. 'size': 2}
  972. def fake_vol_api_roll_detaching(context, volume_id):
  973. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  974. if volumes[volume_id]['status'] == 'detaching':
  975. volumes[volume_id]['status'] = 'in-use'
  976. fake_bdm = fake_block_device.FakeDbBlockDeviceDict(
  977. {'device_name': '/dev/vdb', 'source_type': 'volume',
  978. 'destination_type': 'volume', 'instance_uuid': 'fake',
  979. 'connection_info': '{"foo": "bar"}'})
  980. def fake_vol_api_func(context, volume, *args):
  981. self.assertTrue(uuidutils.is_uuid_like(volume))
  982. return {}
  983. def fake_vol_get(context, volume_id):
  984. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  985. return volumes[volume_id]
  986. def fake_vol_unreserve(context, volume_id):
  987. self.assertTrue(uuidutils.is_uuid_like(volume_id))
  988. if volumes[volume_id]['status'] == 'attaching':
  989. volumes[volume_id]['status'] = 'available'
  990. def fake_vol_migrate_volume_completion(context, old_volume_id,
  991. new_volume_id, error=False):
  992. self.assertTrue(uuidutils.is_uuid_like(old_volume_id))
  993. self.assertTrue(uuidutils.is_uuid_like(new_volume_id))
  994. volumes[old_volume_id]['status'] = 'in-use'
  995. return {'save_volume_id': new_volume_id}
  996. def fake_func_exc(*args, **kwargs):
  997. raise AttributeError # Random exception
  998. def fake_swap_volume(old_connection_info, new_connection_info,
  999. instance, mountpoint, resize_to):
  1000. self.assertEqual(resize_to, 2)
  1001. self.stubs.Set(self.compute.volume_api, 'roll_detaching',
  1002. fake_vol_api_roll_detaching)
  1003. self.stubs.Set(self.compute.volume_api, 'get', fake_vol_get)
  1004. self.stubs.Set(self.compute.volume_api, 'initialize_connection',
  1005. fake_vol_api_func)
  1006. self.stubs.Set(self.compute.volume_api, 'unreserve_volume',
  1007. fake_vol_unreserve)
  1008. self.stubs.Set(self.compute.volume_api, 'terminate_connection',
  1009. fake_vol_api_func)
  1010. self.stubs.Set(db, 'block_device_mapping_get_by_volume_id',
  1011. lambda x, y, z: fake_bdm)
  1012. self.stubs.Set(self.compute.driver, 'get_volume_connector',
  1013. lambda x: {})
  1014. self.stubs.Set(self.compute.driver, 'swap_volume',
  1015. fake_swap_volume)
  1016. self.stubs.Set(self.compute.volume_api, 'migrate_volume_completion',
  1017. fake_vol_migrate_volume_completion)
  1018. self.stubs.Set(db, 'block_device_mapping_update',
  1019. lambda *a, **k: fake_bdm)
  1020. self.stubs.Set(db,
  1021. 'instance_fault_create',
  1022. lambda x, y:
  1023. test_instance_fault.fake_faults['fake-uuid'][0])
  1024. # Good path
  1025. self.compute.swap_volume(self.context, old_volume_id, new_volume_id,
  1026. fake_instance.fake_instance_obj(
  1027. self.context, **{'uuid': 'fake'}))
  1028. self.assertEqual(volumes[old_volume_id]['status'], 'in-use')
  1029. # Error paths
  1030. volumes[old_volume_id]['status'] = 'detaching'
  1031. volumes[new_volume_id]['status'] = 'attaching'
  1032. self.stubs.Set(self.compute.driver, 'swap_volume', fake_func_exc)
  1033. self.assertRaises(AttributeError, self.compute.swap_volume,
  1034. self.context, old_volume_id, new_volume_id,
  1035. fake_instance.fake_instance_obj(
  1036. self.context, **{'uuid': 'fake'}))
  1037. self.assertEqual(volumes[old_volume_id]['status'], 'in-use')
  1038. self.assertEqual(volumes[new_volume_id]['status'], 'available')
  1039. volumes[old_volume_id]['status'] = 'detaching'
  1040. volumes[new_volume_id]['status'] = 'attaching'
  1041. self.stubs.Set(self.compute.volume_api, 'initialize_connection',
  1042. fake_func_exc)
  1043. self.assertRaises(AttributeError, self.compute.swap_volume,
  1044. self.context, old_volume_id, new_volume_id,
  1045. fake_instance.fake_instance_obj(
  1046. self.context, **{'uuid': 'fake'}))
  1047. self.assertEqual(volumes[old_volume_id]['status'], 'in-use')
  1048. self.assertEqual(volumes[new_volume_id]['status'], 'available')
  1049. def test_check_can_live_migrate_source(self):
  1050. is_volume_backed = 'volume_backed'
  1051. dest_check_data = dict(foo='bar')
  1052. db_instance = fake_instance.fake_db_instance()
  1053. instance = objects.Instance._from_db_object(
  1054. self.context, objects.Instance(), db_instance)
  1055. expected_dest_check_data = dict(dest_check_data,
  1056. is_volume_backed=is_volume_backed)
  1057. self.mox.StubOutWithMock(self.compute.compute_api,
  1058. 'is_volume_backed_instance')
  1059. self.mox.StubOutWithMock(self.compute,
  1060. '_get_instance_block_device_info')
  1061. self.mox.StubOutWithMock(self.compute.driver,
  1062. 'check_can_live_migrate_source')
  1063. self.compute.compute_api.is_volume_backed_instance(
  1064. self.context, instance).AndReturn(is_volume_backed)
  1065. self.compute._get_instance_block_device_info(
  1066. self.context, instance, refresh_conn_info=True
  1067. ).AndReturn({'block_device_mapping': 'fake'})
  1068. self.compute.driver.check_can_live_migrate_source(
  1069. self.context, instance, expected_dest_check_data,
  1070. {'block_device_mapping': 'fake'})
  1071. self.mox.ReplayAll()
  1072. self.compute.check_can_live_migrate_source(
  1073. self.context, instance=instance,
  1074. dest_check_data=dest_check_data)
  1075. def _test_check_can_live_migrate_destination(self, do_raise=False,
  1076. has_mig_data=False):
  1077. db_instance = fake_instance.fake_db_instance(host='fake-host')
  1078. instance = objects.Instance._from_db_object(
  1079. self.context, objects.Instance(), db_instance)
  1080. instance.host = 'fake-host'
  1081. block_migration = 'block_migration'
  1082. disk_over_commit = 'disk_over_commit'
  1083. src_info = 'src_info'
  1084. dest_info = 'dest_info'
  1085. dest_check_data = dict(foo='bar')
  1086. mig_data = dict(cow='moo')
  1087. expected_result = dict(mig_data)
  1088. if has_mig_data:
  1089. dest_check_data['migrate_data'] = dict(cat='meow')
  1090. expected_result.update(cat='meow')
  1091. self.mox.StubOutWithMock(self.compute, '_get_compute_info')
  1092. self.mox.StubOutWithMock(self.compute.driver,
  1093. 'check_can_live_migrate_destination')
  1094. self.mox.StubOutWithMock(self.compute.compute_rpcapi,
  1095. 'check_can_live_migrate_source')
  1096. self.mox.StubOutWithMock(self.compute.driver,
  1097. 'check_can_live_migrate_destination_cleanup')
  1098. self.compute._get_compute_info(self.context,
  1099. 'fake-host').AndReturn(src_info)
  1100. self.compute._get_compute_info(self.context,
  1101. CONF.host).AndReturn(dest_info)
  1102. self.compute.driver.check_can_live_migrate_destination(
  1103. self.context, instance, src_info, dest_info,
  1104. block_migration, disk_over_commit).AndReturn(dest_check_data)
  1105. mock_meth = self.compute.compute_rpcapi.check_can_live_migrate_source(
  1106. self.context, instance, dest_check_data)
  1107. if do_raise:
  1108. mock_meth.AndRaise(test.TestingException())
  1109. self.mox.StubOutWithMock(db, 'instance_fault_create')
  1110. db.instance_fault_create(
  1111. self.context, mox.IgnoreArg()).AndReturn(
  1112. test_instance_fault.fake_faults['fake-uuid'][0])
  1113. else:
  1114. mock_meth.AndReturn(mig_data)
  1115. self.compute.driver.check_can_live_migrate_destination_cleanup(
  1116. self.context, dest_check_data)
  1117. self.mox.ReplayAll()
  1118. result = self.compute.check_can_live_migrate_destination(
  1119. self.context, instance=instance,
  1120. block_migration=block_migration,
  1121. disk_over_commit=disk_over_commit)
  1122. self.assertEqual(expected_result, result)
  1123. def test_check_can_live_migrate_destination_success(self):
  1124. self._test_check_can_live_migrate_destination()
  1125. def test_check_can_live_migrate_destination_success_w_mig_data(self):
  1126. self._test_check_can_live_migrate_destination(has_mig_data=True)
  1127. def test_check_can_live_migrate_destination_fail(self):
  1128. self.assertRaises(
  1129. test.TestingException,
  1130. self._test_check_can_live_migrate_destination,
  1131. do_raise=True)
  1132. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  1133. def test_prepare_for_instance_event(self, lock_name_mock):
  1134. inst_obj = objects.Instance(uuid='foo')
  1135. result = self.compute.instance_events.prepare_for_instance_event(
  1136. inst_obj, 'test-event')
  1137. self.assertIn('foo', self.compute.instance_events._events)
  1138. self.assertIn('test-event',
  1139. self.compute.instance_events._events['foo'])
  1140. self.assertEqual(
  1141. result,
  1142. self.compute.instance_events._events['foo']['test-event'])
  1143. self.assertTrue(hasattr(result, 'send'))
  1144. lock_name_mock.assert_called_once_with(inst_obj)
  1145. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  1146. def test_pop_instance_event(self, lock_name_mock):
  1147. event = eventlet_event.Event()
  1148. self.compute.instance_events._events = {
  1149. 'foo': {
  1150. 'test-event': event,
  1151. }
  1152. }
  1153. inst_obj = objects.Instance(uuid='foo')
  1154. event_obj = objects.InstanceExternalEvent(name='test-event',
  1155. tag=None)
  1156. result = self.compute.instance_events.pop_instance_event(inst_obj,
  1157. event_obj)
  1158. self.assertEqual(result, event)
  1159. lock_name_mock.assert_called_once_with(inst_obj)
  1160. @mock.patch('nova.compute.manager.InstanceEvents._lock_name')
  1161. def test_clear_events_for_instance(self, lock_name_mock):
  1162. event = eventlet_event.Event()
  1163. self.compute.instance_events._events = {
  1164. 'foo': {
  1165. 'test-event': event,
  1166. }
  1167. }
  1168. inst_obj = objects.Instance(uuid='foo')
  1169. result = self.compute.instance_events.clear_events_for_instance(
  1170. inst_obj)
  1171. self.assertEqual(result, {'test-event': event})
  1172. lock_name_mock.assert_called_once_with(inst_obj)
  1173. def test_instance_events_lock_name(self):
  1174. inst_obj = objects.Instance(uuid='foo')
  1175. result = self.compute.instance_events._lock_name(inst_obj)
  1176. self.assertEqual(result, 'foo-events')
  1177. def test_prepare_for_instance_event_again(self):
  1178. inst_obj = objects.Instance(uuid='foo')
  1179. self.compute.instance_events.prepare_for_instance_event(
  1180. inst_obj, 'test-event')
  1181. # A second attempt will avoid creating a new list; make sure we
  1182. # get the current list
  1183. result = self.compute.instance_events.prepare_for_instance_event(
  1184. inst_obj, 'test-event')
  1185. self.assertIn('foo', self.compute.instance_events._events)
  1186. self.assertIn('test-event',
  1187. self.compute.instance_events._events['foo'])
  1188. self.assertEqual(
  1189. result,
  1190. self.compute.instance_events._events['foo']['test-event'])
  1191. self.assertTrue(hasattr(result, 'send'))
  1192. def test_process_instance_event(self):
  1193. event = eventlet_event.Event()
  1194. self.compute.instance_events._events = {
  1195. 'foo': {
  1196. 'test-event': event,
  1197. }
  1198. }
  1199. inst_obj = objects.Instance(uuid='foo')
  1200. event_obj = objects.InstanceExternalEvent(name='test-event', tag=None)
  1201. self.compute._process_instance_event(inst_obj, event_obj)
  1202. self.assertTrue(event.ready())
  1203. self.assertEqual(event_obj, event.wait())
  1204. self.assertEqual({}, self.compute.instance_events._events)
  1205. def test_external_instance_event(self):
  1206. instances = [
  1207. objects.Instance(id=1, uuid='uuid1'),
  1208. objects.Instance(id=2, uuid='uuid2')]
  1209. events = [
  1210. objects.InstanceExternalEvent(name='network-changed',
  1211. tag='tag1',
  1212. instance_uuid='uuid1'),
  1213. objects.InstanceExternalEvent(name='foo', instance_uuid='uuid2',
  1214. tag='tag2')]
  1215. @mock.patch.object(self.compute.network_api, 'get_instance_nw_info')
  1216. @mock.patch.object(self.compute, '_process_instance_event')
  1217. def do_test(_process_instance_event, get_instance_nw_info):
  1218. self.compute.external_instance_event(self.context,
  1219. instances, events)
  1220. get_instance_nw_info.assert_called_once_with(self.context,
  1221. instances[0])
  1222. _process_instance_event.assert_called_once_with(instances[1],
  1223. events[1])
  1224. do_test()
  1225. def test_retry_reboot_pending_soft(self):
  1226. instance = objects.Instance(self.context)
  1227. instance.uuid = 'foo'
  1228. instance.task_state = task_states.REBOOT_PENDING
  1229. instance.vm_state = vm_states.ACTIVE
  1230. with mock.patch.object(self.compute, '_get_power_state',
  1231. return_value=power_state.RUNNING):
  1232. allow_reboot, reboot_type = self.compute._retry_reboot(
  1233. context, instance)
  1234. self.assertTrue(allow_reboot)
  1235. self.assertEqual(reboot_type, 'SOFT')
  1236. def test_retry_reboot_pending_hard(self):
  1237. instance = objects.Instance(self.context)
  1238. instance.uuid = 'foo'
  1239. instance.task_state = task_states.REBOOT_PENDING_HARD
  1240. instance.vm_state = vm_states.ACTIVE
  1241. with mock.patch.object(self.compute, '_get_power_state',
  1242. return_value=power_state.RUNNING):
  1243. allow_reboot, reboot_type = self.compute._retry_reboot(
  1244. context, instance)
  1245. self.assertTrue(allow_reboot)
  1246. self.assertEqual(reboot_type, 'HARD')
  1247. def test_retry_reboot_starting_soft_off(self):
  1248. instance = objects.Instance(self.context)
  1249. instance.uuid = 'foo'
  1250. instance.task_state = task_states.REBOOT_STARTED
  1251. with mock.patch.object(self.compute, '_get_power_state',
  1252. return_value=power_state.NOSTATE):
  1253. allow_reboot, reboot_type = self.compute._retry_reboot(
  1254. context, instance)
  1255. self.assertTrue(allow_reboot)
  1256. self.assertEqual(reboot_type, 'HARD')
  1257. def test_retry_reboot_starting_hard_off(self):
  1258. instance = objects.Instance(self.context)
  1259. instance.uuid = 'foo'
  1260. instance.task_state = task_states.REBOOT_STARTED_HARD
  1261. with mock.patch.object(self.compute, '_get_power_state',
  1262. return_value=power_state.NOSTATE):
  1263. allow_reboot, reboot_type = self.compute._retry_reboot(
  1264. context, instance)
  1265. self.assertTrue(allow_reboot)
  1266. self.assertEqual(reboot_type, 'HARD')
  1267. def test_retry_reboot_starting_hard_on(self):
  1268. instance = objects.Instance(self.context)
  1269. instance.uuid = 'foo'
  1270. instance.task_state = task_states.REBOOT_STARTED_HARD
  1271. with mock.patch.object(self.compute, '_get_power_state',
  1272. return_value=power_state.RUNNING):
  1273. allow_reboot, reboot_type = self.compute._retry_reboot(
  1274. context, instance)
  1275. self.assertFalse(allow_reboot)
  1276. self.assertEqual(reboot_type, 'HARD')
  1277. def test_retry_reboot_no_reboot(self):
  1278. instance = objects.Instance(self.context)
  1279. instance.uuid = 'foo'
  1280. instance.task_state = 'bar'
  1281. with mock.patch.object(self.compute, '_get_power_state',
  1282. return_value=power_state.RUNNING):
  1283. allow_reboot, reboot_type = self.compute._retry_reboot(
  1284. context, instance)
  1285. self.assertFalse(allow_reboot)
  1286. self.assertEqual(reboot_type, 'HARD')
  1287. @mock.patch('nova.objects.BlockDeviceMapping.get_by_volume_id')
  1288. @mock.patch('nova.compute.manager.ComputeManager._detach_volume')
  1289. @mock.patch('nova.objects.Instance._from_db_object')
  1290. def test_remove_volume_connection(self, inst_from_db, detach, bdm_get):
  1291. bdm = mock.sentinel.bdm
  1292. inst_obj = mock.sentinel.inst_obj
  1293. bdm_get.return_value = bdm
  1294. inst_from_db.return_value = inst_obj
  1295. with mock.patch.object(self.compute, 'volume_api'):
  1296. self.compute.remove_volume_connection(self.context, 'vol',
  1297. inst_obj)
  1298. detach.assert_called_once_with(self.context, inst_obj, bdm)
  1299. def _test_rescue(self, clean_shutdown=True):
  1300. instance = fake_instance.fake_instance_obj(
  1301. self.context, vm_state=vm_states.ACTIVE)
  1302. fake_nw_info = network_model.NetworkInfo()
  1303. rescue_image_meta = {'id': 'fake', 'name': 'fake'}
  1304. with contextlib.nested(
  1305. mock.patch.object(objects.InstanceActionEvent, 'event_start'),
  1306. mock.patch.object(objects.InstanceActionEvent,
  1307. 'event_finish_with_failure'),
  1308. mock.patch.object(self.context, 'elevated',
  1309. return_value=self.context),
  1310. mock.patch.object(self.compute, '_get_instance_nw_info',
  1311. return_value=fake_nw_info),
  1312. mock.patch.object(self.compute, '_get_rescue_image',
  1313. return_value=rescue_image_meta),
  1314. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  1315. mock.patch.object(self.compute, '_power_off_instance'),
  1316. mock.patch.object(self.compute.driver, 'rescue'),
  1317. mock.patch.object(self.compute.conductor_api,
  1318. 'notify_usage_exists'),
  1319. mock.patch.object(self.compute, '_get_power_state',
  1320. return_value=power_state.RUNNING),
  1321. mock.patch.object(instance, 'save')
  1322. ) as (
  1323. event_start, event_finish, elevated_context, get_nw_info,
  1324. get_rescue_image, notify_instance_usage, power_off_instance,
  1325. driver_rescue, notify_usage_exists, get_power_state, instance_save
  1326. ):
  1327. self.compute.rescue_instance(
  1328. self.context, instance, rescue_password='verybadpass',
  1329. rescue_image_ref=None, clean_shutdown=clean_shutdown)
  1330. # assert the field values on the instance object
  1331. self.assertEqual(vm_states.RESCUED, instance.vm_state)
  1332. self.assertIsNone(instance.task_state)
  1333. self.assertEqual(power_state.RUNNING, instance.power_state)
  1334. self.assertIsNotNone(instance.launched_at)
  1335. # assert our mock calls
  1336. get_nw_info.assert_called_once_with(self.context, instance)
  1337. get_rescue_image.assert_called_once_with(
  1338. self.context, instance, None)
  1339. extra_usage_info = {'rescue_image_name': 'fake'}
  1340. notify_calls = [
  1341. mock.call(self.context, instance, "rescue.start",
  1342. extra_usage_info=extra_usage_info,
  1343. network_info=fake_nw_info),
  1344. mock.call(self.context, instance, "rescue.end",
  1345. extra_usage_info=extra_usage_info,
  1346. network_info=fake_nw_info)
  1347. ]
  1348. notify_instance_usage.assert_has_calls(notify_calls)
  1349. power_off_instance.assert_called_once_with(self.context, instance,
  1350. clean_shutdown)
  1351. driver_rescue.assert_called_once_with(
  1352. self.context, instance, fake_nw_info, rescue_image_meta,
  1353. 'verybadpass')
  1354. notify_usage_exists.assert_called_once_with(
  1355. self.context, instance, current_period=True)
  1356. instance_save.assert_called_once_with(
  1357. expected_task_state=task_states.RESCUING)
  1358. def test_rescue(self):
  1359. self._test_rescue()
  1360. def test_rescue_forced_shutdown(self):
  1361. self._test_rescue(clean_shutdown=False)
  1362. def test_unrescue(self):
  1363. instance = fake_instance.fake_instance_obj(
  1364. self.context, vm_state=vm_states.RESCUED)
  1365. fake_nw_info = network_model.NetworkInfo()
  1366. with contextlib.nested(
  1367. mock.patch.object(objects.InstanceActionEvent, 'event_start'),
  1368. mock.patch.object(objects.InstanceActionEvent,
  1369. 'event_finish_with_failure'),
  1370. mock.patch.object(self.context, 'elevated',
  1371. return_value=self.context),
  1372. mock.patch.object(self.compute, '_get_instance_nw_info',
  1373. return_value=fake_nw_info),
  1374. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  1375. mock.patch.object(self.compute.driver, 'unrescue'),
  1376. mock.patch.object(self.compute, '_get_power_state',
  1377. return_value=power_state.RUNNING),
  1378. mock.patch.object(instance, 'save')
  1379. ) as (
  1380. event_start, event_finish, elevated_context, get_nw_info,
  1381. notify_instance_usage, driver_unrescue, get_power_state,
  1382. instance_save
  1383. ):
  1384. self.compute.unrescue_instance(self.context, instance)
  1385. # assert the field values on the instance object
  1386. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1387. self.assertIsNone(instance.task_state)
  1388. self.assertEqual(power_state.RUNNING, instance.power_state)
  1389. # assert our mock calls
  1390. get_nw_info.assert_called_once_with(self.context, instance)
  1391. notify_calls = [
  1392. mock.call(self.context, instance, "unrescue.start",
  1393. network_info=fake_nw_info),
  1394. mock.call(self.context, instance, "unrescue.end",
  1395. network_info=fake_nw_info)
  1396. ]
  1397. notify_instance_usage.assert_has_calls(notify_calls)
  1398. driver_unrescue.assert_called_once_with(instance, fake_nw_info)
  1399. instance_save.assert_called_once_with(
  1400. expected_task_state=task_states.UNRESCUING)
  1401. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  1402. @mock.patch.object(objects.InstanceActionEvent,
  1403. 'event_finish_with_failure')
  1404. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  1405. return_value=power_state.RUNNING)
  1406. @mock.patch.object(objects.Instance, 'save')
  1407. @mock.patch('nova.utils.generate_password', return_value='fake-pass')
  1408. def test_set_admin_password(self, gen_password_mock,
  1409. instance_save_mock, power_state_mock,
  1410. event_finish_mock, event_start_mock):
  1411. # Ensure instance can have its admin password set.
  1412. instance = fake_instance.fake_instance_obj(
  1413. self.context,
  1414. vm_state=vm_states.ACTIVE,
  1415. task_state=task_states.UPDATING_PASSWORD)
  1416. @mock.patch.object(self.context, 'elevated', return_value=self.context)
  1417. @mock.patch.object(self.compute.driver, 'set_admin_password')
  1418. def do_test(driver_mock, elevated_mock):
  1419. # call the manager method
  1420. self.compute.set_admin_password(self.context, instance, None)
  1421. # make our assertions
  1422. self.assertEqual(vm_states.ACTIVE, instance.vm_state)
  1423. self.assertIsNone(instance.task_state)
  1424. power_state_mock.assert_called_once_with(self.context, instance)
  1425. driver_mock.assert_called_once_with(instance, 'fake-pass')
  1426. instance_save_mock.assert_called_once_with(
  1427. expected_task_state=task_states.UPDATING_PASSWORD)
  1428. do_test()
  1429. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  1430. @mock.patch.object(objects.InstanceActionEvent,
  1431. 'event_finish_with_failure')
  1432. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  1433. return_value=power_state.NOSTATE)
  1434. @mock.patch.object(objects.Instance, 'save')
  1435. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  1436. def test_set_admin_password_bad_state(self, add_fault_mock,
  1437. instance_save_mock, power_state_mock,
  1438. event_finish_mock, event_start_mock):
  1439. # Test setting password while instance is rebuilding.
  1440. instance = fake_instance.fake_instance_obj(self.context)
  1441. with mock.patch.object(self.context, 'elevated',
  1442. return_value=self.context):
  1443. # call the manager method
  1444. self.assertRaises(exception.InstancePasswordSetFailed,
  1445. self.compute.set_admin_password,
  1446. self.context, instance, None)
  1447. # make our assertions
  1448. power_state_mock.assert_called_once_with(self.context, instance)
  1449. instance_save_mock.assert_called_once_with(
  1450. expected_task_state=task_states.UPDATING_PASSWORD)
  1451. add_fault_mock.assert_called_once_with(
  1452. self.context, instance, mock.ANY, mock.ANY)
  1453. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  1454. @mock.patch.object(objects.InstanceActionEvent,
  1455. 'event_finish_with_failure')
  1456. @mock.patch('nova.utils.generate_password', return_value='fake-pass')
  1457. @mock.patch('nova.compute.manager.ComputeManager._get_power_state',
  1458. return_value=power_state.RUNNING)
  1459. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  1460. @mock.patch.object(objects.Instance, 'save')
  1461. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  1462. def _do_test_set_admin_password_driver_error(self, exc,
  1463. expected_vm_state,
  1464. expected_task_state,
  1465. expected_exception,
  1466. add_fault_mock,
  1467. instance_save_mock,
  1468. update_mock,
  1469. power_state_mock,
  1470. gen_password_mock,
  1471. event_finish_mock,
  1472. event_start_mock):
  1473. # Ensure expected exception is raised if set_admin_password fails.
  1474. instance = fake_instance.fake_instance_obj(
  1475. self.context,
  1476. vm_state=vm_states.ACTIVE,
  1477. task_state=task_states.UPDATING_PASSWORD)
  1478. @mock.patch.object(self.context, 'elevated', return_value=self.context)
  1479. @mock.patch.object(self.compute.driver, 'set_admin_password',
  1480. side_effect=exc)
  1481. def do_test(driver_mock, elevated_mock):
  1482. # error raised from the driver should not reveal internal
  1483. # information so a new error is raised
  1484. self.assertRaises(expected_exception,
  1485. self.compute.set_admin_password,
  1486. self.context,
  1487. instance=instance,
  1488. new_pass=None)
  1489. if expected_exception == NotImplementedError:
  1490. instance_save_mock.assert_called_once_with(
  1491. expected_task_state=task_states.UPDATING_PASSWORD)
  1492. else:
  1493. # setting the instance to error state
  1494. instance_save_mock.assert_called_once_with()
  1495. self.assertEqual(expected_vm_state, instance.vm_state)
  1496. # check revert_task_state decorator
  1497. update_mock.assert_called_once_with(
  1498. self.context, instance.uuid,
  1499. task_state=expected_task_state)
  1500. # check wrap_instance_fault decorator
  1501. add_fault_mock.assert_called_once_with(
  1502. self.context, instance, mock.ANY, mock.ANY)
  1503. do_test()
  1504. def test_set_admin_password_driver_not_authorized(self):
  1505. # Ensure expected exception is raised if set_admin_password not
  1506. # authorized.
  1507. exc = exception.Forbidden('Internal error')
  1508. expected_exception = exception.InstancePasswordSetFailed
  1509. self._do_test_set_admin_password_driver_error(
  1510. exc, vm_states.ERROR, None, expected_exception)
  1511. def test_set_admin_password_driver_not_implemented(self):
  1512. # Ensure expected exception is raised if set_admin_password not
  1513. # implemented by driver.
  1514. exc = NotImplementedError()
  1515. expected_exception = NotImplementedError
  1516. self._do_test_set_admin_password_driver_error(
  1517. exc, vm_states.ACTIVE, None, expected_exception)
  1518. def _test_init_host_with_partial_migration(self, task_state=None,
  1519. vm_state=vm_states.ACTIVE):
  1520. our_host = self.compute.host
  1521. instance_1 = objects.Instance(self.context)
  1522. instance_1.uuid = 'foo'
  1523. instance_1.task_state = task_state
  1524. instance_1.vm_state = vm_state
  1525. instance_1.host = 'not-' + our_host
  1526. instance_2 = objects.Instance(self.context)
  1527. instance_2.uuid = 'bar'
  1528. instance_2.task_state = None
  1529. instance_2.vm_state = vm_states.ACTIVE
  1530. instance_2.host = 'not-' + our_host
  1531. with contextlib.nested(
  1532. mock.patch.object(self.compute, '_get_instances_on_driver',
  1533. return_value=[instance_1,
  1534. instance_2]),
  1535. mock.patch.object(self.compute, '_get_instance_nw_info',
  1536. return_value=None),
  1537. mock.patch.object(self.compute, '_get_instance_block_device_info',
  1538. return_value={}),
  1539. mock.patch.object(self.compute, '_is_instance_storage_shared',
  1540. return_value=False),
  1541. mock.patch.object(self.compute.driver, 'destroy')
  1542. ) as (_get_instances_on_driver, _get_instance_nw_info,
  1543. _get_instance_block_device_info, _is_instance_storage_shared,
  1544. destroy):
  1545. self.compute._destroy_evacuated_instances(self.context)
  1546. destroy.assert_called_once_with(self.context, instance_2, None,
  1547. {}, True)
  1548. def test_init_host_with_partial_migration_migrating(self):
  1549. self._test_init_host_with_partial_migration(
  1550. task_state=task_states.MIGRATING)
  1551. def test_init_host_with_partial_migration_resize_migrating(self):
  1552. self._test_init_host_with_partial_migration(
  1553. task_state=task_states.RESIZE_MIGRATING)
  1554. def test_init_host_with_partial_migration_resize_migrated(self):
  1555. self._test_init_host_with_partial_migration(
  1556. task_state=task_states.RESIZE_MIGRATED)
  1557. def test_init_host_with_partial_migration_finish_resize(self):
  1558. self._test_init_host_with_partial_migration(
  1559. task_state=task_states.RESIZE_FINISH)
  1560. def test_init_host_with_partial_migration_resized(self):
  1561. self._test_init_host_with_partial_migration(
  1562. vm_state=vm_states.RESIZED)
  1563. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  1564. def test_error_out_instance_on_exception_not_implemented_err(self,
  1565. inst_update_mock):
  1566. instance = fake_instance.fake_instance_obj(self.context)
  1567. def do_test():
  1568. with self.compute._error_out_instance_on_exception(
  1569. self.context, instance, instance_state=vm_states.STOPPED):
  1570. raise NotImplementedError('test')
  1571. self.assertRaises(NotImplementedError, do_test)
  1572. inst_update_mock.assert_called_once_with(
  1573. self.context, instance.uuid,
  1574. vm_state=vm_states.STOPPED, task_state=None)
  1575. @mock.patch('nova.compute.manager.ComputeManager._instance_update')
  1576. def test_error_out_instance_on_exception_inst_fault_rollback(self,
  1577. inst_update_mock):
  1578. instance = fake_instance.fake_instance_obj(self.context)
  1579. def do_test():
  1580. with self.compute._error_out_instance_on_exception(self.context,
  1581. instance):
  1582. raise exception.InstanceFaultRollback(
  1583. inner_exception=test.TestingException('test'))
  1584. self.assertRaises(test.TestingException, do_test)
  1585. inst_update_mock.assert_called_once_with(
  1586. self.context, instance.uuid,
  1587. vm_state=vm_states.ACTIVE, task_state=None)
  1588. @mock.patch('nova.compute.manager.ComputeManager.'
  1589. '_set_instance_error_state')
  1590. def test_error_out_instance_on_exception_unknown_with_quotas(self,
  1591. set_error):
  1592. instance = fake_instance.fake_instance_obj(self.context)
  1593. quotas = mock.create_autospec(objects.Quotas, spec_set=True)
  1594. def do_test():
  1595. with self.compute._error_out_instance_on_exception(
  1596. self.context, instance, quotas):
  1597. raise test.TestingException('test')
  1598. self.assertRaises(test.TestingException, do_test)
  1599. self.assertEqual(1, len(quotas.method_calls))
  1600. self.assertEqual(mock.call.rollback(), quotas.method_calls[0])
  1601. set_error.assert_called_once_with(self.context, instance)
  1602. def test_cleanup_volumes(self):
  1603. instance = fake_instance.fake_instance_obj(self.context)
  1604. bdm_do_not_delete_dict = fake_block_device.FakeDbBlockDeviceDict(
  1605. {'volume_id': 'fake-id1', 'source_type': 'image',
  1606. 'delete_on_termination': False})
  1607. bdm_delete_dict = fake_block_device.FakeDbBlockDeviceDict(
  1608. {'volume_id': 'fake-id2', 'source_type': 'image',
  1609. 'delete_on_termination': True})
  1610. bdms = block_device_obj.block_device_make_list(self.context,
  1611. [bdm_do_not_delete_dict, bdm_delete_dict])
  1612. with mock.patch.object(self.compute.volume_api,
  1613. 'delete') as volume_delete:
  1614. self.compute._cleanup_volumes(self.context, instance.uuid, bdms)
  1615. volume_delete.assert_called_once_with(self.context,
  1616. bdms[1].volume_id)
  1617. def test_cleanup_volumes_exception_do_not_raise(self):
  1618. instance = fake_instance.fake_instance_obj(self.context)
  1619. bdm_dict1 = fake_block_device.FakeDbBlockDeviceDict(
  1620. {'volume_id': 'fake-id1', 'source_type': 'image',
  1621. 'delete_on_termination': True})
  1622. bdm_dict2 = fake_block_device.FakeDbBlockDeviceDict(
  1623. {'volume_id': 'fake-id2', 'source_type': 'image',
  1624. 'delete_on_termination': True})
  1625. bdms = block_device_obj.block_device_make_list(self.context,
  1626. [bdm_dict1, bdm_dict2])
  1627. with mock.patch.object(self.compute.volume_api,
  1628. 'delete',
  1629. side_effect=[test.TestingException(), None]) as volume_delete:
  1630. self.compute._cleanup_volumes(self.context, instance.uuid, bdms,
  1631. raise_exc=False)
  1632. calls = [mock.call(self.context, bdm.volume_id) for bdm in bdms]
  1633. self.assertEqual(calls, volume_delete.call_args_list)
  1634. def test_cleanup_volumes_exception_raise(self):
  1635. instance = fake_instance.fake_instance_obj(self.context)
  1636. bdm_dict1 = fake_block_device.FakeDbBlockDeviceDict(
  1637. {'volume_id': 'fake-id1', 'source_type': 'image',
  1638. 'delete_on_termination': True})
  1639. bdm_dict2 = fake_block_device.FakeDbBlockDeviceDict(
  1640. {'volume_id': 'fake-id2', 'source_type': 'image',
  1641. 'delete_on_termination': True})
  1642. bdms = block_device_obj.block_device_make_list(self.context,
  1643. [bdm_dict1, bdm_dict2])
  1644. with mock.patch.object(self.compute.volume_api,
  1645. 'delete',
  1646. side_effect=[test.TestingException(), None]) as volume_delete:
  1647. self.assertRaises(test.TestingException,
  1648. self.compute._cleanup_volumes, self.context, instance.uuid,
  1649. bdms)
  1650. calls = [mock.call(self.context, bdm.volume_id) for bdm in bdms]
  1651. self.assertEqual(calls, volume_delete.call_args_list)
  1652. def test_start_building(self):
  1653. instance = fake_instance.fake_instance_obj(self.context)
  1654. with mock.patch.object(self.compute, '_instance_update') as update:
  1655. self.compute._start_building(self.context, instance)
  1656. update.assert_called_once_with(
  1657. self.context, instance.uuid, vm_state=vm_states.BUILDING,
  1658. task_state=None, expected_task_state=(task_states.SCHEDULING,
  1659. None))
  1660. def _test_prebuild_instance_build_abort_exception(self, exc):
  1661. instance = fake_instance.fake_instance_obj(self.context)
  1662. with contextlib.nested(
  1663. mock.patch.object(self.compute, '_check_instance_exists'),
  1664. mock.patch.object(self.compute, '_start_building',
  1665. side_effect=exc)
  1666. ) as (
  1667. check, start
  1668. ):
  1669. # run the code
  1670. self.assertRaises(exception.BuildAbortException,
  1671. self.compute._prebuild_instance,
  1672. self.context, instance)
  1673. # assert the calls
  1674. check.assert_called_once_with(self.context, instance)
  1675. start.assert_called_once_with(self.context, instance)
  1676. def test_prebuild_instance_instance_not_found(self):
  1677. self._test_prebuild_instance_build_abort_exception(
  1678. exception.InstanceNotFound(instance_id='fake'))
  1679. def test_prebuild_instance_unexpected_deleting_task_state_err(self):
  1680. self._test_prebuild_instance_build_abort_exception(
  1681. exception.UnexpectedDeletingTaskStateError(expected='foo',
  1682. actual='bar'))
  1683. def test_stop_instance_task_state_none_power_state_shutdown(self):
  1684. # Tests that stop_instance doesn't puke when the instance power_state
  1685. # is shutdown and the task_state is None.
  1686. instance = fake_instance.fake_instance_obj(
  1687. self.context, vm_state=vm_states.ACTIVE,
  1688. task_state=None, power_state=power_state.SHUTDOWN)
  1689. @mock.patch.object(objects.InstanceActionEvent, 'event_start')
  1690. @mock.patch.object(objects.InstanceActionEvent,
  1691. 'event_finish_with_failure')
  1692. @mock.patch.object(self.compute, '_get_power_state',
  1693. return_value=power_state.SHUTDOWN)
  1694. @mock.patch.object(self.compute, '_notify_about_instance_usage')
  1695. @mock.patch.object(self.compute, '_power_off_instance')
  1696. @mock.patch.object(instance, 'save')
  1697. def do_test(save_mock, power_off_mock, notify_mock, get_state_mock,
  1698. event_finish_mock, event_start_mock):
  1699. # run the code
  1700. self.compute.stop_instance(self.context, instance)
  1701. # assert the calls
  1702. self.assertEqual(2, get_state_mock.call_count)
  1703. notify_mock.assert_has_calls([
  1704. mock.call(self.context, instance, 'power_off.start'),
  1705. mock.call(self.context, instance, 'power_off.end')
  1706. ])
  1707. power_off_mock.assert_called_once_with(
  1708. self.context, instance, True)
  1709. save_mock.assert_called_once_with(
  1710. expected_task_state=[task_states.POWERING_OFF, None])
  1711. self.assertEqual(power_state.SHUTDOWN, instance.power_state)
  1712. self.assertIsNone(instance.task_state)
  1713. self.assertEqual(vm_states.STOPPED, instance.vm_state)
  1714. do_test()
  1715. def test_reset_network_driver_not_implemented(self):
  1716. instance = fake_instance.fake_instance_obj(self.context)
  1717. @mock.patch.object(self.compute.driver, 'reset_network',
  1718. side_effect=NotImplementedError())
  1719. @mock.patch.object(compute_utils, 'add_instance_fault_from_exc')
  1720. def do_test(mock_add_fault, mock_reset):
  1721. self.assertRaises(messaging.ExpectedException,
  1722. self.compute.reset_network,
  1723. self.context,
  1724. instance)
  1725. self.compute = utils.ExceptionHelper(self.compute)
  1726. self.assertRaises(NotImplementedError,
  1727. self.compute.reset_network,
  1728. self.context,
  1729. instance)
  1730. do_test()
  1731. def test_rebuild_default_impl(self):
  1732. def _detach(context, bdms):
  1733. pass
  1734. def _attach(context, instance, bdms, do_check_attach=True):
  1735. return {'block_device_mapping': 'shared_block_storage'}
  1736. def _spawn(context, instance, image_meta, injected_files,
  1737. admin_password, network_info=None, block_device_info=None):
  1738. self.assertEqual(block_device_info['block_device_mapping'],
  1739. 'shared_block_storage')
  1740. with contextlib.nested(
  1741. mock.patch.object(self.compute.driver, 'destroy',
  1742. return_value=None),
  1743. mock.patch.object(self.compute.driver, 'spawn',
  1744. side_effect=_spawn),
  1745. mock.patch.object(objects.Instance, 'save',
  1746. return_value=None)
  1747. ) as(
  1748. mock_destroy,
  1749. mock_spawn,
  1750. mock_save
  1751. ):
  1752. instance = fake_instance.fake_instance_obj(self.context)
  1753. instance.task_state = task_states.REBUILDING
  1754. instance.save(expected_task_state=[task_states.REBUILDING])
  1755. self.compute._rebuild_default_impl(self.context,
  1756. instance,
  1757. None,
  1758. [],
  1759. admin_password='new_pass',
  1760. bdms=[],
  1761. detach_block_devices=_detach,
  1762. attach_block_devices=_attach,
  1763. network_info=None,
  1764. recreate=True,
  1765. block_device_info=None,
  1766. preserve_ephemeral=False)
  1767. self.assertFalse(mock_destroy.called)
  1768. self.assertTrue(mock_save.called)
  1769. self.assertTrue(mock_spawn.called)
  1770. class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
  1771. def setUp(self):
  1772. super(ComputeManagerBuildInstanceTestCase, self).setUp()
  1773. self.compute = importutils.import_object(CONF.compute_manager)
  1774. self.context = context.RequestContext('fake', 'fake')
  1775. self.instance = fake_instance.fake_instance_obj(self.context,
  1776. vm_state=vm_states.ACTIVE,
  1777. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  1778. self.admin_pass = 'pass'
  1779. self.injected_files = []
  1780. self.image = {}
  1781. self.node = 'fake-node'
  1782. self.limits = {}
  1783. self.requested_networks = []
  1784. self.security_groups = []
  1785. self.block_device_mapping = []
  1786. self.filter_properties = {'retry': {'num_attempts': 1,
  1787. 'hosts': [[self.compute.host,
  1788. 'fake-node']]}}
  1789. def fake_network_info():
  1790. return network_model.NetworkInfo()
  1791. self.network_info = network_model.NetworkInfoAsyncWrapper(
  1792. fake_network_info)
  1793. self.block_device_info = self.compute._prep_block_device(context,
  1794. self.instance, self.block_device_mapping)
  1795. # override tracker with a version that doesn't need the database:
  1796. fake_rt = fake_resource_tracker.FakeResourceTracker(self.compute.host,
  1797. self.compute.driver, self.node)
  1798. self.compute._resource_tracker_dict[self.node] = fake_rt
  1799. def _do_build_instance_update(self, reschedule_update=False):
  1800. self.mox.StubOutWithMock(self.instance, 'save')
  1801. self.instance.save(
  1802. expected_task_state=(task_states.SCHEDULING, None)).AndReturn(
  1803. self.instance)
  1804. if reschedule_update:
  1805. self.instance.save().AndReturn(self.instance)
  1806. def _build_and_run_instance_update(self):
  1807. self.mox.StubOutWithMock(self.instance, 'save')
  1808. self._build_resources_instance_update(stub=False)
  1809. self.instance.save(expected_task_state=
  1810. task_states.BLOCK_DEVICE_MAPPING).AndReturn(self.instance)
  1811. def _build_resources_instance_update(self, stub=True):
  1812. if stub:
  1813. self.mox.StubOutWithMock(self.instance, 'save')
  1814. self.instance.save().AndReturn(self.instance)
  1815. def _notify_about_instance_usage(self, event, stub=True, **kwargs):
  1816. if stub:
  1817. self.mox.StubOutWithMock(self.compute,
  1818. '_notify_about_instance_usage')
  1819. self.compute._notify_about_instance_usage(self.context, self.instance,
  1820. event, **kwargs)
  1821. def _instance_action_events(self):
  1822. self.mox.StubOutWithMock(objects.InstanceActionEvent, 'event_start')
  1823. self.mox.StubOutWithMock(objects.InstanceActionEvent,
  1824. 'event_finish_with_failure')
  1825. objects.InstanceActionEvent.event_start(
  1826. self.context, self.instance.uuid, mox.IgnoreArg(),
  1827. want_result=False)
  1828. objects.InstanceActionEvent.event_finish_with_failure(
  1829. self.context, self.instance.uuid, mox.IgnoreArg(),
  1830. exc_val=mox.IgnoreArg(), exc_tb=mox.IgnoreArg(),
  1831. want_result=False)
  1832. @mock.patch('nova.utils.spawn_n')
  1833. def test_build_and_run_instance_called_with_proper_args(self, mock_spawn):
  1834. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  1835. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  1836. self._do_build_instance_update()
  1837. self.compute._build_and_run_instance(self.context, self.instance,
  1838. self.image, self.injected_files, self.admin_pass,
  1839. self.requested_networks, self.security_groups,
  1840. self.block_device_mapping, self.node, self.limits,
  1841. self.filter_properties)
  1842. self._instance_action_events()
  1843. self.mox.ReplayAll()
  1844. self.compute.build_and_run_instance(self.context, self.instance,
  1845. self.image, request_spec={},
  1846. filter_properties=self.filter_properties,
  1847. injected_files=self.injected_files,
  1848. admin_password=self.admin_pass,
  1849. requested_networks=self.requested_networks,
  1850. security_groups=self.security_groups,
  1851. block_device_mapping=self.block_device_mapping, node=self.node,
  1852. limits=self.limits)
  1853. # This test when sending an icehouse compatible rpc call to juno compute
  1854. # node, NetworkRequest object can load from three items tuple.
  1855. @mock.patch('nova.objects.InstanceActionEvent.event_finish_with_failure')
  1856. @mock.patch('nova.objects.InstanceActionEvent.event_start')
  1857. @mock.patch('nova.objects.Instance.save')
  1858. @mock.patch('nova.compute.manager.ComputeManager._build_and_run_instance')
  1859. @mock.patch('nova.utils.spawn_n')
  1860. def test_build_and_run_instance_with_icehouse_requested_network(
  1861. self, mock_spawn, mock_build_and_run, mock_save, mock_event_start,
  1862. mock_event_finish):
  1863. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  1864. mock_save.return_value = self.instance
  1865. self.compute.build_and_run_instance(self.context, self.instance,
  1866. self.image, request_spec={},
  1867. filter_properties=self.filter_properties,
  1868. injected_files=self.injected_files,
  1869. admin_password=self.admin_pass,
  1870. requested_networks=[('fake_network_id', '10.0.0.1',
  1871. 'fake_port_id')],
  1872. security_groups=self.security_groups,
  1873. block_device_mapping=self.block_device_mapping, node=self.node,
  1874. limits=self.limits)
  1875. requested_network = mock_build_and_run.call_args[0][5][0]
  1876. self.assertEqual('fake_network_id', requested_network.network_id)
  1877. self.assertEqual('10.0.0.1', str(requested_network.address))
  1878. self.assertEqual('fake_port_id', requested_network.port_id)
  1879. @mock.patch('nova.utils.spawn_n')
  1880. def test_build_abort_exception(self, mock_spawn):
  1881. def fake_spawn(f, *args, **kwargs):
  1882. # NOTE(danms): Simulate the detached nature of spawn so that
  1883. # we confirm that the inner task has the fault logic
  1884. try:
  1885. return f(*args, **kwargs)
  1886. except Exception:
  1887. pass
  1888. mock_spawn.side_effect = fake_spawn
  1889. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  1890. self.mox.StubOutWithMock(self.compute, '_cleanup_allocated_networks')
  1891. self.mox.StubOutWithMock(self.compute, '_cleanup_volumes')
  1892. self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
  1893. self.mox.StubOutWithMock(self.compute, '_set_instance_error_state')
  1894. self.mox.StubOutWithMock(self.compute.compute_task_api,
  1895. 'build_instances')
  1896. self._do_build_instance_update()
  1897. self.compute._build_and_run_instance(self.context, self.instance,
  1898. self.image, self.injected_files, self.admin_pass,
  1899. self.requested_networks, self.security_groups,
  1900. self.block_device_mapping, self.node, self.limits,
  1901. self.filter_properties).AndRaise(
  1902. exception.BuildAbortException(reason='',
  1903. instance_uuid=self.instance.uuid))
  1904. self.compute._cleanup_allocated_networks(self.context, self.instance,
  1905. self.requested_networks)
  1906. self.compute._cleanup_volumes(self.context, self.instance.uuid,
  1907. self.block_device_mapping, raise_exc=False)
  1908. compute_utils.add_instance_fault_from_exc(self.context,
  1909. self.instance, mox.IgnoreArg(), mox.IgnoreArg())
  1910. self.compute._set_instance_error_state(self.context, self.instance)
  1911. self._instance_action_events()
  1912. self.mox.ReplayAll()
  1913. self.compute.build_and_run_instance(self.context, self.instance,
  1914. self.image, request_spec={},
  1915. filter_properties=self.filter_properties,
  1916. injected_files=self.injected_files,
  1917. admin_password=self.admin_pass,
  1918. requested_networks=self.requested_networks,
  1919. security_groups=self.security_groups,
  1920. block_device_mapping=self.block_device_mapping, node=self.node,
  1921. limits=self.limits)
  1922. @mock.patch('nova.utils.spawn_n')
  1923. def test_rescheduled_exception(self, mock_spawn):
  1924. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  1925. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  1926. self.mox.StubOutWithMock(self.compute, '_set_instance_error_state')
  1927. self.mox.StubOutWithMock(self.compute.compute_task_api,
  1928. 'build_instances')
  1929. self._do_build_instance_update(reschedule_update=True)
  1930. self.compute._build_and_run_instance(self.context, self.instance,
  1931. self.image, self.injected_files, self.admin_pass,
  1932. self.requested_networks, self.security_groups,
  1933. self.block_device_mapping, self.node, self.limits,
  1934. self.filter_properties).AndRaise(
  1935. exception.RescheduledException(reason='',
  1936. instance_uuid=self.instance.uuid))
  1937. self.compute.compute_task_api.build_instances(self.context,
  1938. [self.instance], self.image, self.filter_properties,
  1939. self.admin_pass, self.injected_files, self.requested_networks,
  1940. self.security_groups, self.block_device_mapping)
  1941. self._instance_action_events()
  1942. self.mox.ReplayAll()
  1943. self.compute.build_and_run_instance(self.context, self.instance,
  1944. self.image, request_spec={},
  1945. filter_properties=self.filter_properties,
  1946. injected_files=self.injected_files,
  1947. admin_password=self.admin_pass,
  1948. requested_networks=self.requested_networks,
  1949. security_groups=self.security_groups,
  1950. block_device_mapping=self.block_device_mapping, node=self.node,
  1951. limits=self.limits)
  1952. def test_rescheduled_exception_with_non_ascii_exception(self):
  1953. exc = exception.NovaException(u's\xe9quence')
  1954. self.mox.StubOutWithMock(self.compute.driver, 'spawn')
  1955. self.mox.StubOutWithMock(conductor_rpcapi.ConductorAPI,
  1956. 'instance_update')
  1957. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  1958. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  1959. self.compute._build_networks_for_instance(self.context, self.instance,
  1960. self.requested_networks, self.security_groups).AndReturn(
  1961. self.network_info)
  1962. self.compute._shutdown_instance(self.context, self.instance,
  1963. self.block_device_mapping, self.requested_networks,
  1964. try_deallocate_networks=False)
  1965. self._notify_about_instance_usage('create.start',
  1966. extra_usage_info={'image_name': self.image.get('name')})
  1967. self._build_and_run_instance_update()
  1968. self.compute.driver.spawn(self.context, self.instance, self.image,
  1969. self.injected_files, self.admin_pass,
  1970. network_info=self.network_info,
  1971. block_device_info=self.block_device_info).AndRaise(exc)
  1972. self._notify_about_instance_usage('create.error',
  1973. fault=exc, stub=False)
  1974. conductor_rpcapi.ConductorAPI.instance_update(
  1975. self.context, self.instance['uuid'], mox.IgnoreArg(), 'conductor')
  1976. self.mox.ReplayAll()
  1977. self.assertRaises(exception.RescheduledException,
  1978. self.compute._build_and_run_instance, self.context,
  1979. self.instance, self.image, self.injected_files,
  1980. self.admin_pass, self.requested_networks, self.security_groups,
  1981. self.block_device_mapping, self.node,
  1982. self.limits, self.filter_properties)
  1983. @mock.patch('nova.utils.spawn_n')
  1984. def test_rescheduled_exception_without_retry(self, mock_spawn):
  1985. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  1986. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  1987. self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
  1988. self.mox.StubOutWithMock(self.compute, '_set_instance_error_state')
  1989. self.mox.StubOutWithMock(self.compute, '_cleanup_allocated_networks')
  1990. self.mox.StubOutWithMock(self.compute, '_cleanup_volumes')
  1991. self._do_build_instance_update()
  1992. self.compute._build_and_run_instance(self.context, self.instance,
  1993. self.image, self.injected_files, self.admin_pass,
  1994. self.requested_networks, self.security_groups,
  1995. self.block_device_mapping, self.node, self.limits,
  1996. {}).AndRaise(
  1997. exception.RescheduledException(reason='',
  1998. instance_uuid=self.instance.uuid))
  1999. self.compute._cleanup_allocated_networks(self.context, self.instance,
  2000. self.requested_networks)
  2001. compute_utils.add_instance_fault_from_exc(self.context, self.instance,
  2002. mox.IgnoreArg(), mox.IgnoreArg())
  2003. self.compute._set_instance_error_state(self.context,
  2004. self.instance)
  2005. self._instance_action_events()
  2006. self.mox.ReplayAll()
  2007. self.compute.build_and_run_instance(self.context, self.instance,
  2008. self.image, request_spec={},
  2009. filter_properties={},
  2010. injected_files=self.injected_files,
  2011. admin_password=self.admin_pass,
  2012. requested_networks=self.requested_networks,
  2013. security_groups=self.security_groups,
  2014. block_device_mapping=self.block_device_mapping, node=self.node,
  2015. limits=self.limits)
  2016. @mock.patch('nova.utils.spawn_n')
  2017. def test_rescheduled_exception_do_not_deallocate_network(self, mock_spawn):
  2018. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  2019. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  2020. self.mox.StubOutWithMock(self.compute.driver,
  2021. 'deallocate_networks_on_reschedule')
  2022. self.mox.StubOutWithMock(self.compute, '_cleanup_allocated_networks')
  2023. self.mox.StubOutWithMock(self.compute.compute_task_api,
  2024. 'build_instances')
  2025. self._do_build_instance_update(reschedule_update=True)
  2026. self.compute._build_and_run_instance(self.context, self.instance,
  2027. self.image, self.injected_files, self.admin_pass,
  2028. self.requested_networks, self.security_groups,
  2029. self.block_device_mapping, self.node, self.limits,
  2030. self.filter_properties).AndRaise(
  2031. exception.RescheduledException(reason='',
  2032. instance_uuid=self.instance.uuid))
  2033. self.compute.driver.deallocate_networks_on_reschedule(
  2034. self.instance).AndReturn(False)
  2035. self.compute.compute_task_api.build_instances(self.context,
  2036. [self.instance], self.image, self.filter_properties,
  2037. self.admin_pass, self.injected_files, self.requested_networks,
  2038. self.security_groups, self.block_device_mapping)
  2039. self._instance_action_events()
  2040. self.mox.ReplayAll()
  2041. self.compute.build_and_run_instance(self.context, self.instance,
  2042. self.image, request_spec={},
  2043. filter_properties=self.filter_properties,
  2044. injected_files=self.injected_files,
  2045. admin_password=self.admin_pass,
  2046. requested_networks=self.requested_networks,
  2047. security_groups=self.security_groups,
  2048. block_device_mapping=self.block_device_mapping, node=self.node,
  2049. limits=self.limits)
  2050. @mock.patch('nova.utils.spawn_n')
  2051. def test_rescheduled_exception_deallocate_network(self, mock_spawn):
  2052. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  2053. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  2054. self.mox.StubOutWithMock(self.compute.driver,
  2055. 'deallocate_networks_on_reschedule')
  2056. self.mox.StubOutWithMock(self.compute, '_cleanup_allocated_networks')
  2057. self.mox.StubOutWithMock(self.compute.compute_task_api,
  2058. 'build_instances')
  2059. self._do_build_instance_update(reschedule_update=True)
  2060. self.compute._build_and_run_instance(self.context, self.instance,
  2061. self.image, self.injected_files, self.admin_pass,
  2062. self.requested_networks, self.security_groups,
  2063. self.block_device_mapping, self.node, self.limits,
  2064. self.filter_properties).AndRaise(
  2065. exception.RescheduledException(reason='',
  2066. instance_uuid=self.instance.uuid))
  2067. self.compute.driver.deallocate_networks_on_reschedule(
  2068. self.instance).AndReturn(True)
  2069. self.compute._cleanup_allocated_networks(self.context, self.instance,
  2070. self.requested_networks)
  2071. self.compute.compute_task_api.build_instances(self.context,
  2072. [self.instance], self.image, self.filter_properties,
  2073. self.admin_pass, self.injected_files, self.requested_networks,
  2074. self.security_groups, self.block_device_mapping)
  2075. self._instance_action_events()
  2076. self.mox.ReplayAll()
  2077. self.compute.build_and_run_instance(self.context, self.instance,
  2078. self.image, request_spec={},
  2079. filter_properties=self.filter_properties,
  2080. injected_files=self.injected_files,
  2081. admin_password=self.admin_pass,
  2082. requested_networks=self.requested_networks,
  2083. security_groups=self.security_groups,
  2084. block_device_mapping=self.block_device_mapping, node=self.node,
  2085. limits=self.limits)
  2086. def _test_build_and_run_exceptions(self, exc, set_error=False,
  2087. cleanup_volumes=False):
  2088. self.mox.StubOutWithMock(self.compute, '_build_and_run_instance')
  2089. self.mox.StubOutWithMock(self.compute, '_cleanup_allocated_networks')
  2090. self.mox.StubOutWithMock(self.compute, '_cleanup_volumes')
  2091. self.mox.StubOutWithMock(self.compute.compute_task_api,
  2092. 'build_instances')
  2093. self._do_build_instance_update()
  2094. self.compute._build_and_run_instance(self.context, self.instance,
  2095. self.image, self.injected_files, self.admin_pass,
  2096. self.requested_networks, self.security_groups,
  2097. self.block_device_mapping, self.node, self.limits,
  2098. self.filter_properties).AndRaise(exc)
  2099. self.compute._cleanup_allocated_networks(self.context, self.instance,
  2100. self.requested_networks)
  2101. if cleanup_volumes:
  2102. self.compute._cleanup_volumes(self.context, self.instance.uuid,
  2103. self.block_device_mapping, raise_exc=False)
  2104. if set_error:
  2105. self.mox.StubOutWithMock(self.compute, '_set_instance_error_state')
  2106. self.mox.StubOutWithMock(compute_utils,
  2107. 'add_instance_fault_from_exc')
  2108. compute_utils.add_instance_fault_from_exc(self.context,
  2109. self.instance, mox.IgnoreArg(), mox.IgnoreArg())
  2110. self.compute._set_instance_error_state(self.context, self.instance)
  2111. self._instance_action_events()
  2112. self.mox.ReplayAll()
  2113. with mock.patch('nova.utils.spawn_n') as mock_spawn:
  2114. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  2115. self.compute.build_and_run_instance(self.context, self.instance,
  2116. self.image, request_spec={},
  2117. filter_properties=self.filter_properties,
  2118. injected_files=self.injected_files,
  2119. admin_password=self.admin_pass,
  2120. requested_networks=self.requested_networks,
  2121. security_groups=self.security_groups,
  2122. block_device_mapping=self.block_device_mapping, node=self.node,
  2123. limits=self.limits)
  2124. def test_build_and_run_notfound_exception(self):
  2125. self._test_build_and_run_exceptions(exception.InstanceNotFound(
  2126. instance_id=''))
  2127. def test_build_and_run_unexpecteddeleting_exception(self):
  2128. self._test_build_and_run_exceptions(
  2129. exception.UnexpectedDeletingTaskStateError(expected='',
  2130. actual=''))
  2131. def test_build_and_run_buildabort_exception(self):
  2132. self._test_build_and_run_exceptions(exception.BuildAbortException(
  2133. instance_uuid='', reason=''), set_error=True, cleanup_volumes=True)
  2134. def test_build_and_run_unhandled_exception(self):
  2135. self._test_build_and_run_exceptions(test.TestingException(),
  2136. set_error=True, cleanup_volumes=True)
  2137. def test_instance_not_found(self):
  2138. exc = exception.InstanceNotFound(instance_id=1)
  2139. self.mox.StubOutWithMock(self.compute.driver, 'spawn')
  2140. self.mox.StubOutWithMock(conductor_rpcapi.ConductorAPI,
  2141. 'instance_update')
  2142. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2143. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  2144. self.compute._build_networks_for_instance(self.context, self.instance,
  2145. self.requested_networks, self.security_groups).AndReturn(
  2146. self.network_info)
  2147. self.compute._shutdown_instance(self.context, self.instance,
  2148. self.block_device_mapping, self.requested_networks,
  2149. try_deallocate_networks=False)
  2150. self._notify_about_instance_usage('create.start',
  2151. extra_usage_info={'image_name': self.image.get('name')})
  2152. self._build_and_run_instance_update()
  2153. self.compute.driver.spawn(self.context, self.instance, self.image,
  2154. self.injected_files, self.admin_pass,
  2155. network_info=self.network_info,
  2156. block_device_info=self.block_device_info).AndRaise(exc)
  2157. self._notify_about_instance_usage('create.end',
  2158. fault=exc, stub=False)
  2159. conductor_rpcapi.ConductorAPI.instance_update(
  2160. self.context, self.instance.uuid, mox.IgnoreArg(), 'conductor')
  2161. self.mox.ReplayAll()
  2162. self.assertRaises(exception.InstanceNotFound,
  2163. self.compute._build_and_run_instance, self.context,
  2164. self.instance, self.image, self.injected_files,
  2165. self.admin_pass, self.requested_networks, self.security_groups,
  2166. self.block_device_mapping, self.node,
  2167. self.limits, self.filter_properties)
  2168. def test_reschedule_on_exception(self):
  2169. self.mox.StubOutWithMock(self.compute.driver, 'spawn')
  2170. self.mox.StubOutWithMock(conductor_rpcapi.ConductorAPI,
  2171. 'instance_update')
  2172. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2173. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  2174. self.compute._build_networks_for_instance(self.context, self.instance,
  2175. self.requested_networks, self.security_groups).AndReturn(
  2176. self.network_info)
  2177. self.compute._shutdown_instance(self.context, self.instance,
  2178. self.block_device_mapping, self.requested_networks,
  2179. try_deallocate_networks=False)
  2180. self._notify_about_instance_usage('create.start',
  2181. extra_usage_info={'image_name': self.image.get('name')})
  2182. self._build_and_run_instance_update()
  2183. exc = test.TestingException()
  2184. self.compute.driver.spawn(self.context, self.instance, self.image,
  2185. self.injected_files, self.admin_pass,
  2186. network_info=self.network_info,
  2187. block_device_info=self.block_device_info).AndRaise(exc)
  2188. conductor_rpcapi.ConductorAPI.instance_update(
  2189. self.context, self.instance.uuid, mox.IgnoreArg(), 'conductor')
  2190. self._notify_about_instance_usage('create.error',
  2191. fault=exc, stub=False)
  2192. self.mox.ReplayAll()
  2193. self.assertRaises(exception.RescheduledException,
  2194. self.compute._build_and_run_instance, self.context,
  2195. self.instance, self.image, self.injected_files,
  2196. self.admin_pass, self.requested_networks, self.security_groups,
  2197. self.block_device_mapping, self.node,
  2198. self.limits, self.filter_properties)
  2199. def test_spawn_network_alloc_failure(self):
  2200. # Because network allocation is asynchronous, failures may not present
  2201. # themselves until the virt spawn method is called.
  2202. self._test_build_and_run_spawn_exceptions(exception.NoMoreNetworks())
  2203. def test_build_and_run_flavor_disk_too_small_exception(self):
  2204. self._test_build_and_run_spawn_exceptions(
  2205. exception.FlavorDiskTooSmall())
  2206. def test_build_and_run_flavor_memory_too_small_exception(self):
  2207. self._test_build_and_run_spawn_exceptions(
  2208. exception.FlavorMemoryTooSmall())
  2209. def test_build_and_run_image_not_active_exception(self):
  2210. self._test_build_and_run_spawn_exceptions(
  2211. exception.ImageNotActive(image_id=self.image.get('id')))
  2212. def test_build_and_run_image_unacceptable_exception(self):
  2213. self._test_build_and_run_spawn_exceptions(
  2214. exception.ImageUnacceptable(image_id=self.image.get('id'),
  2215. reason=""))
  2216. def _test_build_and_run_spawn_exceptions(self, exc):
  2217. with contextlib.nested(
  2218. mock.patch.object(self.compute.driver, 'spawn',
  2219. side_effect=exc),
  2220. mock.patch.object(conductor_rpcapi.ConductorAPI,
  2221. 'instance_update'),
  2222. mock.patch.object(self.instance, 'save',
  2223. side_effect=[self.instance, self.instance]),
  2224. mock.patch.object(self.compute,
  2225. '_build_networks_for_instance',
  2226. return_value=self.network_info),
  2227. mock.patch.object(self.compute,
  2228. '_notify_about_instance_usage'),
  2229. mock.patch.object(self.compute,
  2230. '_shutdown_instance'),
  2231. mock.patch.object(self.compute,
  2232. '_validate_instance_group_policy')
  2233. ) as (spawn, instance_update, save,
  2234. _build_networks_for_instance, _notify_about_instance_usage,
  2235. _shutdown_instance, _validate_instance_group_policy):
  2236. self.assertRaises(exception.BuildAbortException,
  2237. self.compute._build_and_run_instance, self.context,
  2238. self.instance, self.image, self.injected_files,
  2239. self.admin_pass, self.requested_networks,
  2240. self.security_groups, self.block_device_mapping, self.node,
  2241. self.limits, self.filter_properties)
  2242. _validate_instance_group_policy.assert_called_once_with(
  2243. self.context, self.instance, self.filter_properties)
  2244. _build_networks_for_instance.assert_has_calls(
  2245. mock.call(self.context, self.instance,
  2246. self.requested_networks, self.security_groups))
  2247. _notify_about_instance_usage.assert_has_calls([
  2248. mock.call(self.context, self.instance, 'create.start',
  2249. extra_usage_info={'image_name': self.image.get('name')}),
  2250. mock.call(self.context, self.instance, 'create.error',
  2251. fault=exc)])
  2252. save.assert_has_calls([
  2253. mock.call(),
  2254. mock.call(
  2255. expected_task_state=task_states.BLOCK_DEVICE_MAPPING)])
  2256. spawn.assert_has_calls(mock.call(self.context, self.instance,
  2257. self.image, self.injected_files, self.admin_pass,
  2258. network_info=self.network_info,
  2259. block_device_info=self.block_device_info))
  2260. instance_update.assert_has_calls(mock.call(self.context,
  2261. self.instance.uuid, mock.ANY, 'conductor'))
  2262. _shutdown_instance.assert_called_once_with(self.context,
  2263. self.instance, self.block_device_mapping,
  2264. self.requested_networks, try_deallocate_networks=False)
  2265. @mock.patch('nova.compute.manager.ComputeManager._get_power_state')
  2266. def test_spawn_waits_for_network_and_saves_info_cache(self, gps):
  2267. inst = mock.MagicMock()
  2268. network_info = mock.MagicMock()
  2269. with mock.patch.object(self.compute, 'driver'):
  2270. self.compute._spawn(self.context, inst, {}, network_info, None,
  2271. None, None)
  2272. network_info.wait.assert_called_once_with(do_raise=True)
  2273. self.assertEqual(network_info, inst.info_cache.network_info)
  2274. inst.save.assert_called_with(expected_task_state=task_states.SPAWNING)
  2275. @mock.patch('nova.utils.spawn_n')
  2276. def test_reschedule_on_resources_unavailable(self, mock_spawn):
  2277. mock_spawn.side_effect = lambda f, *a, **k: f(*a, **k)
  2278. reason = 'resource unavailable'
  2279. exc = exception.ComputeResourcesUnavailable(reason=reason)
  2280. class FakeResourceTracker(object):
  2281. def instance_claim(self, context, instance, limits):
  2282. raise exc
  2283. self.mox.StubOutWithMock(self.compute, '_get_resource_tracker')
  2284. self.mox.StubOutWithMock(self.compute.compute_task_api,
  2285. 'build_instances')
  2286. self.compute._get_resource_tracker(self.node).AndReturn(
  2287. FakeResourceTracker())
  2288. self._do_build_instance_update(reschedule_update=True)
  2289. self._notify_about_instance_usage('create.start',
  2290. extra_usage_info={'image_name': self.image.get('name')})
  2291. self._notify_about_instance_usage('create.error',
  2292. fault=exc, stub=False)
  2293. self.compute.compute_task_api.build_instances(self.context,
  2294. [self.instance], self.image, self.filter_properties,
  2295. self.admin_pass, self.injected_files, self.requested_networks,
  2296. self.security_groups, self.block_device_mapping)
  2297. self._instance_action_events()
  2298. self.mox.ReplayAll()
  2299. self.compute.build_and_run_instance(self.context, self.instance,
  2300. self.image, request_spec={},
  2301. filter_properties=self.filter_properties,
  2302. injected_files=self.injected_files,
  2303. admin_password=self.admin_pass,
  2304. requested_networks=self.requested_networks,
  2305. security_groups=self.security_groups,
  2306. block_device_mapping=self.block_device_mapping, node=self.node,
  2307. limits=self.limits)
  2308. def test_build_resources_buildabort_reraise(self):
  2309. exc = exception.BuildAbortException(
  2310. instance_uuid=self.instance.uuid, reason='')
  2311. self.mox.StubOutWithMock(self.compute, '_build_resources')
  2312. self.mox.StubOutWithMock(conductor_rpcapi.ConductorAPI,
  2313. 'instance_update')
  2314. conductor_rpcapi.ConductorAPI.instance_update(
  2315. self.context, self.instance.uuid, mox.IgnoreArg(), 'conductor')
  2316. self._notify_about_instance_usage('create.start',
  2317. extra_usage_info={'image_name': self.image.get('name')})
  2318. self.compute._build_resources(self.context, self.instance,
  2319. self.requested_networks, self.security_groups, self.image,
  2320. self.block_device_mapping).AndRaise(exc)
  2321. self._notify_about_instance_usage('create.error',
  2322. fault=exc, stub=False)
  2323. self.mox.ReplayAll()
  2324. self.assertRaises(exception.BuildAbortException,
  2325. self.compute._build_and_run_instance, self.context,
  2326. self.instance, self.image, self.injected_files,
  2327. self.admin_pass, self.requested_networks,
  2328. self.security_groups, self.block_device_mapping, self.node,
  2329. self.limits, self.filter_properties)
  2330. def test_build_resources_reraises_on_failed_bdm_prep(self):
  2331. self.mox.StubOutWithMock(self.compute, '_prep_block_device')
  2332. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2333. self.compute._build_networks_for_instance(self.context, self.instance,
  2334. self.requested_networks, self.security_groups).AndReturn(
  2335. self.network_info)
  2336. self._build_resources_instance_update()
  2337. self.compute._prep_block_device(self.context, self.instance,
  2338. self.block_device_mapping).AndRaise(test.TestingException())
  2339. self.mox.ReplayAll()
  2340. try:
  2341. with self.compute._build_resources(self.context, self.instance,
  2342. self.requested_networks, self.security_groups,
  2343. self.image, self.block_device_mapping):
  2344. pass
  2345. except Exception as e:
  2346. self.assertIsInstance(e, exception.BuildAbortException)
  2347. def test_failed_bdm_prep_from_delete_raises_unexpected(self):
  2348. with contextlib.nested(
  2349. mock.patch.object(self.compute,
  2350. '_build_networks_for_instance',
  2351. return_value=self.network_info),
  2352. mock.patch.object(self.instance, 'save',
  2353. side_effect=exception.UnexpectedDeletingTaskStateError(
  2354. actual=task_states.DELETING, expected='None')),
  2355. ) as (_build_networks_for_instance, save):
  2356. try:
  2357. with self.compute._build_resources(self.context, self.instance,
  2358. self.requested_networks, self.security_groups,
  2359. self.image, self.block_device_mapping):
  2360. pass
  2361. except Exception as e:
  2362. self.assertIsInstance(e,
  2363. exception.UnexpectedDeletingTaskStateError)
  2364. _build_networks_for_instance.assert_has_calls(
  2365. mock.call(self.context, self.instance,
  2366. self.requested_networks, self.security_groups))
  2367. save.assert_has_calls(mock.call())
  2368. def test_build_resources_aborts_on_failed_network_alloc(self):
  2369. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2370. self.compute._build_networks_for_instance(self.context, self.instance,
  2371. self.requested_networks, self.security_groups).AndRaise(
  2372. test.TestingException())
  2373. self.mox.ReplayAll()
  2374. try:
  2375. with self.compute._build_resources(self.context, self.instance,
  2376. self.requested_networks, self.security_groups, self.image,
  2377. self.block_device_mapping):
  2378. pass
  2379. except Exception as e:
  2380. self.assertIsInstance(e, exception.BuildAbortException)
  2381. def test_failed_network_alloc_from_delete_raises_unexpected(self):
  2382. with mock.patch.object(self.compute,
  2383. '_build_networks_for_instance') as _build_networks:
  2384. exc = exception.UnexpectedDeletingTaskStateError
  2385. _build_networks.side_effect = exc(actual=task_states.DELETING,
  2386. expected='None')
  2387. try:
  2388. with self.compute._build_resources(self.context, self.instance,
  2389. self.requested_networks, self.security_groups,
  2390. self.image, self.block_device_mapping):
  2391. pass
  2392. except Exception as e:
  2393. self.assertIsInstance(e, exc)
  2394. _build_networks.assert_has_calls(
  2395. mock.call(self.context, self.instance,
  2396. self.requested_networks, self.security_groups))
  2397. def test_build_resources_with_network_info_obj_on_spawn_failure(self):
  2398. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2399. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  2400. self.compute._build_networks_for_instance(self.context, self.instance,
  2401. self.requested_networks, self.security_groups).AndReturn(
  2402. network_model.NetworkInfo())
  2403. self.compute._shutdown_instance(self.context, self.instance,
  2404. self.block_device_mapping, self.requested_networks,
  2405. try_deallocate_networks=False)
  2406. self._build_resources_instance_update()
  2407. self.mox.ReplayAll()
  2408. test_exception = test.TestingException()
  2409. def fake_spawn():
  2410. raise test_exception
  2411. try:
  2412. with self.compute._build_resources(self.context, self.instance,
  2413. self.requested_networks, self.security_groups,
  2414. self.image, self.block_device_mapping):
  2415. fake_spawn()
  2416. except Exception as e:
  2417. self.assertEqual(test_exception, e)
  2418. def test_build_resources_cleans_up_and_reraises_on_spawn_failure(self):
  2419. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2420. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  2421. self.compute._build_networks_for_instance(self.context, self.instance,
  2422. self.requested_networks, self.security_groups).AndReturn(
  2423. self.network_info)
  2424. self.compute._shutdown_instance(self.context, self.instance,
  2425. self.block_device_mapping, self.requested_networks,
  2426. try_deallocate_networks=False)
  2427. self._build_resources_instance_update()
  2428. self.mox.ReplayAll()
  2429. test_exception = test.TestingException()
  2430. def fake_spawn():
  2431. raise test_exception
  2432. try:
  2433. with self.compute._build_resources(self.context, self.instance,
  2434. self.requested_networks, self.security_groups,
  2435. self.image, self.block_device_mapping):
  2436. fake_spawn()
  2437. except Exception as e:
  2438. self.assertEqual(test_exception, e)
  2439. @mock.patch('nova.network.model.NetworkInfoAsyncWrapper.wait')
  2440. @mock.patch(
  2441. 'nova.compute.manager.ComputeManager._build_networks_for_instance')
  2442. @mock.patch('nova.objects.Instance.save')
  2443. def test_build_resources_instance_not_found_before_yield(
  2444. self, mock_save, mock_build_network, mock_info_wait):
  2445. mock_build_network.return_value = self.network_info
  2446. expected_exc = exception.InstanceNotFound(
  2447. instance_id=self.instance.uuid)
  2448. mock_save.side_effect = expected_exc
  2449. try:
  2450. with self.compute._build_resources(self.context, self.instance,
  2451. self.requested_networks, self.security_groups,
  2452. self.image, self.block_device_mapping):
  2453. raise
  2454. except Exception as e:
  2455. self.assertEqual(expected_exc, e)
  2456. mock_build_network.assert_called_once_with(self.context, self.instance,
  2457. self.requested_networks, self.security_groups)
  2458. mock_info_wait.assert_called_once_with(do_raise=False)
  2459. @mock.patch('nova.network.model.NetworkInfoAsyncWrapper.wait')
  2460. @mock.patch(
  2461. 'nova.compute.manager.ComputeManager._build_networks_for_instance')
  2462. @mock.patch('nova.objects.Instance.save')
  2463. def test_build_resources_unexpected_task_error_before_yield(
  2464. self, mock_save, mock_build_network, mock_info_wait):
  2465. mock_build_network.return_value = self.network_info
  2466. mock_save.side_effect = exception.UnexpectedTaskStateError(
  2467. expected='', actual='')
  2468. try:
  2469. with self.compute._build_resources(self.context, self.instance,
  2470. self.requested_networks, self.security_groups,
  2471. self.image, self.block_device_mapping):
  2472. raise
  2473. except exception.BuildAbortException:
  2474. pass
  2475. mock_build_network.assert_called_once_with(self.context, self.instance,
  2476. self.requested_networks, self.security_groups)
  2477. mock_info_wait.assert_called_once_with(do_raise=False)
  2478. @mock.patch('nova.network.model.NetworkInfoAsyncWrapper.wait')
  2479. @mock.patch(
  2480. 'nova.compute.manager.ComputeManager._build_networks_for_instance')
  2481. @mock.patch('nova.objects.Instance.save')
  2482. def test_build_resources_exception_before_yield(
  2483. self, mock_save, mock_build_network, mock_info_wait):
  2484. mock_build_network.return_value = self.network_info
  2485. mock_save.side_effect = Exception()
  2486. try:
  2487. with self.compute._build_resources(self.context, self.instance,
  2488. self.requested_networks, self.security_groups,
  2489. self.image, self.block_device_mapping):
  2490. raise
  2491. except exception.BuildAbortException:
  2492. pass
  2493. mock_build_network.assert_called_once_with(self.context, self.instance,
  2494. self.requested_networks, self.security_groups)
  2495. mock_info_wait.assert_called_once_with(do_raise=False)
  2496. def test_build_resources_aborts_on_cleanup_failure(self):
  2497. self.mox.StubOutWithMock(self.compute, '_build_networks_for_instance')
  2498. self.mox.StubOutWithMock(self.compute, '_shutdown_instance')
  2499. self.compute._build_networks_for_instance(self.context, self.instance,
  2500. self.requested_networks, self.security_groups).AndReturn(
  2501. self.network_info)
  2502. self.compute._shutdown_instance(self.context, self.instance,
  2503. self.block_device_mapping, self.requested_networks,
  2504. try_deallocate_networks=False).AndRaise(
  2505. test.TestingException())
  2506. self._build_resources_instance_update()
  2507. self.mox.ReplayAll()
  2508. def fake_spawn():
  2509. raise test.TestingException()
  2510. try:
  2511. with self.compute._build_resources(self.context, self.instance,
  2512. self.requested_networks, self.security_groups,
  2513. self.image, self.block_device_mapping):
  2514. fake_spawn()
  2515. except Exception as e:
  2516. self.assertIsInstance(e, exception.BuildAbortException)
  2517. def test_build_networks_if_not_allocated(self):
  2518. instance = fake_instance.fake_instance_obj(self.context,
  2519. system_metadata={},
  2520. expected_attrs=['system_metadata'])
  2521. self.mox.StubOutWithMock(self.compute, '_get_instance_nw_info')
  2522. self.mox.StubOutWithMock(self.compute, '_allocate_network')
  2523. self.compute._allocate_network(self.context, instance,
  2524. self.requested_networks, None, self.security_groups, None)
  2525. self.mox.ReplayAll()
  2526. self.compute._build_networks_for_instance(self.context, instance,
  2527. self.requested_networks, self.security_groups)
  2528. def test_build_networks_if_allocated_false(self):
  2529. instance = fake_instance.fake_instance_obj(self.context,
  2530. system_metadata=dict(network_allocated='False'),
  2531. expected_attrs=['system_metadata'])
  2532. self.mox.StubOutWithMock(self.compute, '_get_instance_nw_info')
  2533. self.mox.StubOutWithMock(self.compute, '_allocate_network')
  2534. self.compute._allocate_network(self.context, instance,
  2535. self.requested_networks, None, self.security_groups, None)
  2536. self.mox.ReplayAll()
  2537. self.compute._build_networks_for_instance(self.context, instance,
  2538. self.requested_networks, self.security_groups)
  2539. def test_return_networks_if_found(self):
  2540. instance = fake_instance.fake_instance_obj(self.context,
  2541. system_metadata=dict(network_allocated='True'),
  2542. expected_attrs=['system_metadata'])
  2543. def fake_network_info():
  2544. return network_model.NetworkInfo([{'address': '123.123.123.123'}])
  2545. self.mox.StubOutWithMock(self.compute, '_get_instance_nw_info')
  2546. self.mox.StubOutWithMock(self.compute, '_allocate_network')
  2547. self.compute._get_instance_nw_info(self.context, instance).AndReturn(
  2548. network_model.NetworkInfoAsyncWrapper(fake_network_info))
  2549. self.mox.ReplayAll()
  2550. self.compute._build_networks_for_instance(self.context, instance,
  2551. self.requested_networks, self.security_groups)
  2552. def test_cleanup_allocated_networks_instance_not_found(self):
  2553. with contextlib.nested(
  2554. mock.patch.object(self.compute, '_deallocate_network'),
  2555. mock.patch.object(self.instance, 'save',
  2556. side_effect=exception.InstanceNotFound(instance_id=''))
  2557. ) as (_deallocate_network, save):
  2558. # Testing that this doesn't raise an exeption
  2559. self.compute._cleanup_allocated_networks(self.context,
  2560. self.instance, self.requested_networks)
  2561. save.assert_called_once_with()
  2562. self.assertEqual('False',
  2563. self.instance.system_metadata['network_allocated'])
  2564. @mock.patch.object(conductor_rpcapi.ConductorAPI, 'instance_update')
  2565. def test_launched_at_in_create_end_notification(self,
  2566. mock_instance_update):
  2567. def fake_notify(*args, **kwargs):
  2568. if args[2] == 'create.end':
  2569. # Check that launched_at is set on the instance
  2570. self.assertIsNotNone(args[1].launched_at)
  2571. with contextlib.nested(
  2572. mock.patch.object(self.compute.driver, 'spawn'),
  2573. mock.patch.object(self.compute,
  2574. '_build_networks_for_instance', return_value=[]),
  2575. mock.patch.object(self.instance, 'save'),
  2576. mock.patch.object(self.compute, '_notify_about_instance_usage',
  2577. side_effect=fake_notify)
  2578. ) as (mock_spawn, mock_networks, mock_save, mock_notify):
  2579. self.compute._build_and_run_instance(self.context, self.instance,
  2580. self.image, self.injected_files, self.admin_pass,
  2581. self.requested_networks, self.security_groups,
  2582. self.block_device_mapping, self.node, self.limits,
  2583. self.filter_properties)
  2584. expected_call = mock.call(self.context, self.instance,
  2585. 'create.end', extra_usage_info={'message': u'Success'},
  2586. network_info=[])
  2587. create_end_call = mock_notify.call_args_list[
  2588. mock_notify.call_count - 1]
  2589. self.assertEqual(expected_call, create_end_call)
  2590. @mock.patch.object(conductor_rpcapi.ConductorAPI, 'instance_update')
  2591. def test_create_end_on_instance_delete(self, mock_instance_update):
  2592. def fake_notify(*args, **kwargs):
  2593. if args[2] == 'create.end':
  2594. # Check that launched_at is set on the instance
  2595. self.assertIsNotNone(args[1].launched_at)
  2596. exc = exception.InstanceNotFound(instance_id='')
  2597. with contextlib.nested(
  2598. mock.patch.object(self.compute.driver, 'spawn'),
  2599. mock.patch.object(self.compute,
  2600. '_build_networks_for_instance', return_value=[]),
  2601. mock.patch.object(self.instance, 'save',
  2602. side_effect=[None, None, exc]),
  2603. mock.patch.object(self.compute, '_notify_about_instance_usage',
  2604. side_effect=fake_notify)
  2605. ) as (mock_spawn, mock_networks, mock_save, mock_notify):
  2606. self.assertRaises(exception.InstanceNotFound,
  2607. self.compute._build_and_run_instance, self.context,
  2608. self.instance, self.image, self.injected_files,
  2609. self.admin_pass, self.requested_networks,
  2610. self.security_groups, self.block_device_mapping, self.node,
  2611. self.limits, self.filter_properties)
  2612. expected_call = mock.call(self.context, self.instance,
  2613. 'create.end', fault=exc)
  2614. create_end_call = mock_notify.call_args_list[
  2615. mock_notify.call_count - 1]
  2616. self.assertEqual(expected_call, create_end_call)
  2617. class ComputeManagerMigrationTestCase(test.NoDBTestCase):
  2618. def setUp(self):
  2619. super(ComputeManagerMigrationTestCase, self).setUp()
  2620. self.compute = importutils.import_object(CONF.compute_manager)
  2621. self.context = context.RequestContext('fake', 'fake')
  2622. self.image = {}
  2623. self.instance = fake_instance.fake_instance_obj(self.context,
  2624. vm_state=vm_states.ACTIVE,
  2625. expected_attrs=['metadata', 'system_metadata', 'info_cache'])
  2626. self.migration = objects.Migration()
  2627. self.migration.status = 'migrating'
  2628. def test_finish_resize_failure(self):
  2629. elevated_context = self.context.elevated()
  2630. with contextlib.nested(
  2631. mock.patch.object(self.compute, '_finish_resize',
  2632. side_effect=exception.ResizeError(reason='')),
  2633. mock.patch.object(objects.InstanceActionEvent, 'event_start'),
  2634. mock.patch.object(objects.InstanceActionEvent,
  2635. 'event_finish_with_failure'),
  2636. mock.patch.object(db, 'instance_fault_create'),
  2637. mock.patch.object(self.compute, '_instance_update'),
  2638. mock.patch.object(self.migration, 'save'),
  2639. mock.patch.object(self.context, 'elevated',
  2640. return_value=elevated_context)
  2641. ) as (meth, event_start, event_finish, fault_create, instance_update,
  2642. migration_save, context_elevated):
  2643. fault_create.return_value = (
  2644. test_instance_fault.fake_faults['fake-uuid'][0])
  2645. self.assertRaises(
  2646. exception.ResizeError, self.compute.finish_resize,
  2647. context=self.context, disk_info=[], image=self.image,
  2648. instance=self.instance, reservations=[],
  2649. migration=self.migration
  2650. )
  2651. self.assertEqual("error", self.migration.status)
  2652. migration_save.assert_has_calls([mock.call(elevated_context)])
  2653. def test_resize_instance_failure(self):
  2654. elevated_context = self.context.elevated()
  2655. self.migration.dest_host = None
  2656. with contextlib.nested(
  2657. mock.patch.object(self.compute.driver,
  2658. 'migrate_disk_and_power_off',
  2659. side_effect=exception.ResizeError(reason='')),
  2660. mock.patch.object(objects.InstanceActionEvent, 'event_start'),
  2661. mock.patch.object(objects.InstanceActionEvent,
  2662. 'event_finish_with_failure'),
  2663. mock.patch.object(db, 'instance_fault_create'),
  2664. mock.patch.object(self.compute, '_instance_update'),
  2665. mock.patch.object(self.migration, 'save'),
  2666. mock.patch.object(self.context, 'elevated',
  2667. return_value=elevated_context),
  2668. mock.patch.object(self.compute, '_get_instance_nw_info',
  2669. return_value=None),
  2670. mock.patch.object(self.instance, 'save'),
  2671. mock.patch.object(self.compute, '_notify_about_instance_usage'),
  2672. mock.patch.object(self.compute,
  2673. '_get_instance_block_device_info',
  2674. return_value=None),
  2675. mock.patch.object(objects.BlockDeviceMappingList,
  2676. 'get_by_instance_uuid',
  2677. return_value=None)
  2678. ) as (meth, event_start, event_finish, fault_create, instance_update,
  2679. migration_save, context_elevated, nw_info, save_inst, notify,
  2680. vol_block_info, bdm):
  2681. fault_create.return_value = (
  2682. test_instance_fault.fake_faults['fake-uuid'][0])
  2683. self.assertRaises(
  2684. exception.ResizeError, self.compute.resize_instance,
  2685. context=self.context, instance=self.instance, image=self.image,
  2686. reservations=[], migration=self.migration, instance_type='type'
  2687. )
  2688. self.assertEqual("error", self.migration.status)
  2689. migration_save.assert_has_calls([mock.call(elevated_context)])