Convert NFV unit tests from nose to stestr

stestr is the current openstack test framework.
The unit tests are encapsulated within test classes
so that global variables are not required.

Some unit tests use fixtures while others are only mock.
Tox targets based off unit tests such as coverage
have also been updated.

Story: 2003499
Task: 26531
Change-Id: Iaa95dcc592ee5ef17deb88f94e8f15239910e7ce
Signed-off-by: Al Bailey <Al.Bailey@windriver.com>
changes/34/603434/2
Al Bailey 4 years ago
parent 76a213442d
commit 2d39affda2
  1. 1
      .gitignore
  2. 7
      nfv/.coveragerc
  3. 4
      nfv/.stestr.conf
  4. 15
      nfv/coverage.ini
  5. 0
      nfv/nfv-tests/__init__.py
  6. 4
      nfv/nfv-tests/nfv_scenario_tests/tests/_alarms.py
  7. 0
      nfv/nfv-tests/nfv_unit_tests/tests/__init__.py
  8. 38
      nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py
  9. 596
      nfv/nfv-tests/nfv_unit_tests/tests/test_instance.py
  10. 745
      nfv/nfv-tests/nfv_unit_tests/tests/test_instance_director.py
  11. 1167
      nfv/nfv-tests/nfv_unit_tests/tests/test_sw_patch_strategy.py
  12. 946
      nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py
  13. 12
      nfv/nfv-tests/nfv_unit_tests/tests/testcase.py
  14. 43
      nfv/tox.ini

1
.gitignore vendored

@ -9,6 +9,7 @@
.*.swp
.coverage
.installed.cfg
.stestr
.testrepository
.tox
.venv

@ -0,0 +1,7 @@
[run]
branch = True
source = nfv_common,nfv_client,nfv_vim,nfv_plugins
omit = nfv-tests/*
[report]
ignore_errors = True

@ -0,0 +1,4 @@
[DEFAULT]
test_path=./nfv-tests/nfv_unit_tests/tests
top_dir=.

@ -1,15 +0,0 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
[run]
branch = True
source =
nfv-common
nfv-plugins
nfv-vim
[report]
include =
*nfv_vim/directors/_instance_director.py

@ -71,7 +71,7 @@ def is_instance_reboot_alarm(alarms, instance, guest_hb=False):
"""
Check if an instance reboot alarm has been raised
"""
expected_alarm = {'alarm_id': fm_constants.FM_ALARM_ID_VM_REBOOT,
expected_alarm = {'alarm_id': fm_constants.FM_ALARM_ID_VM_REBOOTING,
'severity': fm_constants.FM_ALARM_SEVERITY_CRITICAL}
return _instance_alarm_raised(alarms, expected_alarm, instance)
@ -81,7 +81,7 @@ def is_instance_rebuild_alarm(alarms, instance, guest_hb=False):
"""
Check if an instance rebuild alarm has been raised
"""
expected_alarm = {'alarm_id': fm_constants.FM_ALARM_ID_VM_REBUILD,
expected_alarm = {'alarm_id': fm_constants.FM_ALARM_ID_VM_REBUILDING,
'severity': fm_constants.FM_ALARM_SEVERITY_CRITICAL}
return _instance_alarm_raised(alarms, expected_alarm, instance)

@ -9,25 +9,29 @@ import subprocess
from nfv_vim import database
from nfv_vim import tables
from . import testcase
def test_nfv_vim_database_upgrade_from_18_03():
"""
Test VIM database upgrades from 18.03 GA
"""
root_dir = os.environ['VIRTUAL_ENV']
devnull = open(os.devnull, 'w')
try:
vim_cmd = ("nfv-vim-manage db-load-data -d %s "
"-f %s/nfv_vim_db_18.03_GA" % (root_dir, root_dir))
class TestNFVDatabaseUpgrade(testcase.NFVTestCase):
subprocess.check_call([vim_cmd], shell=True, stderr=devnull)
def test_nfv_vim_database_upgrade_from_18_03(self):
"""
Test VIM database upgrades from 18.03 GA
"""
root_dir = os.environ['VIRTUAL_ENV']
except subprocess.CalledProcessError:
raise
devnull = open(os.devnull, 'w')
try:
vim_cmd = ("nfv-vim-manage db-load-data -d %s "
"-f %s/nfv_vim_db_18.03_GA" % (root_dir, root_dir))
config = dict()
config['database_dir'] = root_dir
database.database_initialize(config)
database.database_migrate_data()
tables.tables_initialize()
subprocess.check_call([vim_cmd], shell=True, stderr=devnull)
except subprocess.CalledProcessError:
raise
config = dict()
config['database_dir'] = root_dir
database.database_initialize(config)
database.database_migrate_data()
tables.tables_initialize()

@ -3,8 +3,8 @@
#
# SPDX-License-Identifier: Apache-2.0
#
import fixtures
import uuid
import mock
from nfv_common import config
@ -24,342 +24,324 @@ from nfv_vim.tables._instance_group_table import InstanceGroupTable
from nfv_vim.nfvi.objects import v1 as nfvi_objects
import utils
# Constants
# Globals
_tenant_table = Table()
_instance_type_table = InstanceTypeTable()
_image_table = ImageTable()
_instance_table = InstanceTable()
_instance_group_table = InstanceGroupTable()
_host_table = HostTable()
_host_group_table = HostGroupTable()
_host_aggregate_table = HostAggregateTable()
# Don't attempt to write to the database while unit testing
_tenant_table.persist = False
_image_table.persist = False
_instance_type_table.persist = False
_instance_table.persist = False
_instance_group_table.persist = False
_host_table.persist = False
_host_group_table.persist = False
_host_aggregate_table.persist = False
def create_instance(instance_name, instance_type_name, image_name, host_name,
admin_state=nfvi.objects.v1.INSTANCE_ADMIN_STATE.UNLOCKED,
live_migration_timeout=None):
"""
Create an instance
"""
global _tenant_table, _instance_table, _image_table
tenant_uuid = str(uuid.uuid4())
tenant = objects.Tenant(tenant_uuid, "%s_name" % tenant_uuid, '', True)
_tenant_table[tenant_uuid] = tenant
for instance_type in _instance_type_table.values():
if instance_type.name == instance_type_name:
for image in _image_table.values():
if image.name == image_name:
instance_uuid = str(uuid.uuid4())
nfvi_instance = nfvi.objects.v1.Instance(
instance_uuid, instance_name, tenant_uuid,
admin_state=admin_state,
oper_state=nfvi.objects.v1.INSTANCE_OPER_STATE.ENABLED,
avail_status=list(),
action=nfvi.objects.v1.INSTANCE_ACTION.NONE,
host_name=host_name,
instance_type=utils.instance_type_to_flavor_dict(
instance_type),
image_uuid=image.uuid,
live_migration_timeout=live_migration_timeout)
instance = objects.Instance(nfvi_instance)
_instance_table[instance.uuid] = instance
return instance
assert 0, "Unknown instance_type_name: %s" % instance_type_name
def create_instance_type(instance_type_name, live_migration_timeout=None):
"""
Create an instance type
"""
global _instance_type_table
instance_type_uuid = str(uuid.uuid4())
instance_type = objects.InstanceType(instance_type_uuid,
instance_type_name)
instance_type.update_details(
vcpus=1, mem_mb=64, disk_gb=1, ephemeral_gb=0,
swap_gb=0, guest_services=None,
auto_recovery=True,
live_migration_timeout=live_migration_timeout,
live_migration_max_downtime=500,
storage_type='local_image')
_instance_type_table[instance_type_uuid] = instance_type
def create_image(image_name, properties=None):
"""
Create an instance type
"""
global _image_table
image_uuid = str(uuid.uuid4())
nfvi_image = nfvi_objects.Image(image_uuid, image_name, 'description',
nfvi_objects.IMAGE_AVAIL_STATUS.AVAILABLE,
nfvi_objects.IMAGE_ACTION.NONE,
'BARE',
'QCOW2',
1,
64,
'public',
False,
properties=properties)
image = objects.Image(nfvi_image)
_image_table[image_uuid] = image
def create_instance_group(name, members, policies):
"""
Create an instance group
"""
global _instance_group_table
member_uuids = []
for instance_uuid, instance in _instance_table.iteritems():
if instance.name in members:
member_uuids.append(instance_uuid)
nfvi_instance_group = nfvi.objects.v1.InstanceGroup(
uuid=str(uuid.uuid4()),
name=name,
member_uuids=member_uuids,
policies=policies
)
instance_group = objects.InstanceGroup(nfvi_instance_group)
_instance_group_table[instance_group.uuid] = instance_group
def create_host(host_name,
cpe=False,
admin_state=nfvi.objects.v1.HOST_ADMIN_STATE.UNLOCKED,
software_load='12.01',
target_load='12.01'):
"""
Create a host
"""
global _host_table
personality = ''
if host_name.startswith('controller'):
personality = HOST_PERSONALITY.CONTROLLER
if cpe:
personality = personality + ',' + HOST_PERSONALITY.COMPUTE
elif host_name.startswith('compute'):
personality = HOST_PERSONALITY.COMPUTE
elif host_name.startswith('storage'):
personality = HOST_PERSONALITY.STORAGE
else:
assert 0, "Invalid host_name: %s" % host_name
nfvi_host = nfvi.objects.v1.Host(
uuid=str(uuid.uuid4()),
name=host_name,
personality=personality,
admin_state=admin_state,
oper_state=nfvi.objects.v1.HOST_OPER_STATE.ENABLED,
avail_status=nfvi.objects.v1.HOST_AVAIL_STATUS.AVAILABLE,
action=nfvi.objects.v1.HOST_ACTION.NONE,
software_load=software_load,
target_load=target_load,
uptime='1000'
)
host = objects.Host(nfvi_host,
initial_state=host_fsm.HOST_STATE.ENABLED)
_host_table[host.name] = host
def create_host_group(name, members, policies):
"""
Create a host group
"""
global _host_group_table
member_uuids = []
for instance_uuid, instance in _instance_table.iteritems():
if instance.name in members:
member_uuids.append(instance_uuid)
nfvi_host_group = nfvi.objects.v1.HostGroup(
name=name,
member_names=members,
policies=policies
)
host_group = objects.HostGroup(nfvi_host_group)
_host_group_table[host_group.name] = host_group
def create_host_aggregate(name, host_names):
"""
Create a host aggregate
"""
global _host_aggregate_table
nfvi_host_aggregate = nfvi.objects.v1.HostAggregate(
name=name,
host_names=host_names,
availability_zone=''
)
host_aggregate = objects.HostAggregate(nfvi_host_aggregate)
_host_aggregate_table[host_aggregate.name] = host_aggregate
from . import testcase
from . import utils
def fake_event_issue(a, b, c, d):
"""
Mock out the _event_issue function because it is being called when instance
objects are created. It ends up trying to communicate with another thread
(that doesn't exist) and this eventually leads to nosetests hanging if
(that doesn't exist) and this eventually leads to unit tests hanging if
enough events are issued.
"""
return None
@mock.patch('nfv_vim.event_log._instance._event_issue', fake_event_issue)
@mock.patch('nfv_vim.tables._tenant_table._tenant_table', _tenant_table)
@mock.patch('nfv_vim.tables._host_table._host_table', _host_table)
@mock.patch('nfv_vim.tables._instance_group_table._instance_group_table', _instance_group_table)
@mock.patch('nfv_vim.tables._host_group_table._host_group_table', _host_group_table)
@mock.patch('nfv_vim.tables._host_aggregate_table._host_aggregate_table', _host_aggregate_table)
@mock.patch('nfv_vim.tables._instance_table._instance_table', _instance_table)
@mock.patch('nfv_vim.tables._instance_type_table._instance_type_table', _instance_type_table)
@mock.patch('nfv_vim.tables._image_table._image_table', _image_table)
class TestInstance(object):
def setup(self):
# NOTE: The following testcases test the same scenarios as the testcases in
# nova/tests/unit/virt/libvirt/test_driver.py
class TestInstance(testcase.NFVTestCase):
_tenant_table = Table()
_instance_type_table = InstanceTypeTable()
_image_table = ImageTable()
_instance_table = InstanceTable()
_instance_group_table = InstanceGroupTable()
_host_table = HostTable()
_host_group_table = HostGroupTable()
_host_aggregate_table = HostAggregateTable()
# Don't attempt to write to the database while unit testing
_tenant_table.persist = False
_image_table.persist = False
_instance_type_table.persist = False
_instance_table.persist = False
_instance_group_table.persist = False
_host_table.persist = False
_host_group_table.persist = False
_host_aggregate_table.persist = False
def setUp(self):
"""
Setup for testing.
"""
pass
def teardown(self):
super(TestInstance, self).setUp()
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._tenant_table._tenant_table',
self._tenant_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._host_table._host_table',
self._host_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._instance_group_table._instance_group_table',
self._instance_group_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._host_group_table._host_group_table',
self._host_group_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._host_aggregate_table._host_aggregate_table',
self._host_aggregate_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._instance_table._instance_table',
self._instance_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._instance_type_table._instance_type_table',
self._instance_type_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.tables._image_table._image_table',
self._image_table))
self.useFixture(fixtures.MonkeyPatch('nfv_vim.event_log._instance._event_issue',
fake_event_issue))
def tearDown(self):
"""
Cleanup testing setup.
"""
_tenant_table.clear()
_instance_type_table.clear()
_image_table.clear()
_instance_table.clear()
_instance_group_table.clear()
_host_table.clear()
_host_group_table.clear()
_host_aggregate_table.clear()
# NOTE: The following testcases test the same scenarios as the testcases in
# nova/tests/unit/virt/libvirt/test_driver.py
super(TestInstance, self).tearDown()
self._tenant_table.clear()
self._instance_type_table.clear()
self._image_table.clear()
self._instance_table.clear()
self._instance_group_table.clear()
self._host_table.clear()
self._host_group_table.clear()
self._host_aggregate_table.clear()
def create_instance(self, instance_name, instance_type_name, image_name, host_name,
admin_state=nfvi.objects.v1.INSTANCE_ADMIN_STATE.UNLOCKED,
live_migration_timeout=None):
"""
Create an instance
"""
tenant_uuid = str(uuid.uuid4())
tenant = objects.Tenant(tenant_uuid, "%s_name" % tenant_uuid, '', True)
self._tenant_table[tenant_uuid] = tenant
for instance_type in self._instance_type_table.values():
if instance_type.name == instance_type_name:
for image in self._image_table.values():
if image.name == image_name:
instance_uuid = str(uuid.uuid4())
nfvi_instance = nfvi.objects.v1.Instance(
instance_uuid, instance_name, tenant_uuid,
admin_state=admin_state,
oper_state=nfvi.objects.v1.INSTANCE_OPER_STATE.ENABLED,
avail_status=list(),
action=nfvi.objects.v1.INSTANCE_ACTION.NONE,
host_name=host_name,
instance_type=utils.instance_type_to_flavor_dict(
instance_type),
image_uuid=image.uuid,
live_migration_timeout=live_migration_timeout)
instance = objects.Instance(nfvi_instance)
self._instance_table[instance.uuid] = instance
return instance
assert 0, "Unknown instance_type_name: %s" % instance_type_name
def create_instance_type(self, instance_type_name, live_migration_timeout=None):
"""
Create an instance type
"""
instance_type_uuid = str(uuid.uuid4())
instance_type = objects.InstanceType(instance_type_uuid,
instance_type_name)
instance_type.update_details(
vcpus=1, mem_mb=64, disk_gb=1, ephemeral_gb=0,
swap_gb=0, guest_services=None,
auto_recovery=True,
live_migration_timeout=live_migration_timeout,
live_migration_max_downtime=500,
storage_type='local_image')
self._instance_type_table[instance_type_uuid] = instance_type
def create_image(self, image_name, properties=None):
"""
Create an image
"""
image_uuid = str(uuid.uuid4())
nfvi_image = nfvi_objects.Image(image_uuid, image_name, 'description',
nfvi_objects.IMAGE_AVAIL_STATUS.AVAILABLE,
nfvi_objects.IMAGE_ACTION.NONE,
'BARE',
'QCOW2',
1,
64,
'public',
False,
properties=properties)
image = objects.Image(nfvi_image)
self._image_table[image_uuid] = image
def create_instance_group(self, name, members, policies):
"""
Create an instance group
"""
member_uuids = []
for instance_uuid, instance in self._instance_table.iteritems():
if instance.name in members:
member_uuids.append(instance_uuid)
nfvi_instance_group = nfvi.objects.v1.InstanceGroup(
uuid=str(uuid.uuid4()),
name=name,
member_uuids=member_uuids,
policies=policies
)
instance_group = objects.InstanceGroup(nfvi_instance_group)
self._instance_group_table[instance_group.uuid] = instance_group
def create_host(self, host_name,
cpe=False,
admin_state=nfvi.objects.v1.HOST_ADMIN_STATE.UNLOCKED,
software_load='12.01',
target_load='12.01'):
"""
Create a host
"""
personality = ''
if host_name.startswith('controller'):
personality = HOST_PERSONALITY.CONTROLLER
if cpe:
personality = personality + ',' + HOST_PERSONALITY.COMPUTE
elif host_name.startswith('compute'):
personality = HOST_PERSONALITY.COMPUTE
elif host_name.startswith('storage'):
personality = HOST_PERSONALITY.STORAGE
else:
assert 0, "Invalid host_name: %s" % host_name
nfvi_host = nfvi.objects.v1.Host(
uuid=str(uuid.uuid4()),
name=host_name,
personality=personality,
admin_state=admin_state,
oper_state=nfvi.objects.v1.HOST_OPER_STATE.ENABLED,
avail_status=nfvi.objects.v1.HOST_AVAIL_STATUS.AVAILABLE,
action=nfvi.objects.v1.HOST_ACTION.NONE,
software_load=software_load,
target_load=target_load,
uptime='1000'
)
host = objects.Host(nfvi_host,
initial_state=host_fsm.HOST_STATE.ENABLED)
self._host_table[host.name] = host
def create_host_group(self, name, members, policies):
"""
Create a host group
"""
member_uuids = []
for instance_uuid, instance in self._instance_table.iteritems():
if instance.name in members:
member_uuids.append(instance_uuid)
nfvi_host_group = nfvi.objects.v1.HostGroup(
name=name,
member_names=members,
policies=policies
)
host_group = objects.HostGroup(nfvi_host_group)
self._host_group_table[host_group.name] = host_group
def create_host_aggregate(self, name, host_names):
"""
Create a host aggregate
"""
nfvi_host_aggregate = nfvi.objects.v1.HostAggregate(
name=name,
host_names=host_names,
availability_zone=''
)
host_aggregate = objects.HostAggregate(nfvi_host_aggregate)
self._host_aggregate_table[host_aggregate.name] = host_aggregate
def test_live_migration_completion_timeout(self):
create_instance_type('small')
create_image('image_0')
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small')
self.create_image('image_0')
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 800 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_from_flavor(self):
create_instance_type('small', live_migration_timeout=300)
create_image('image_0')
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small', live_migration_timeout=300)
self.create_image('image_0')
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 300 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_from_image(self):
create_instance_type('small')
create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "400"})
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small')
self.create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "400"})
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 400 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_from_instance(self):
create_instance_type('small')
create_image('image_0')
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=200)
self.create_instance_type('small')
self.create_image('image_0')
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=200)
assert 200 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_flavor_overwrite_image(self):
create_instance_type('small', live_migration_timeout=300)
create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "400"})
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small', live_migration_timeout=300)
self.create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "400"})
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 300 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_image_overwrite_flavor(self):
create_instance_type('small', live_migration_timeout=300)
create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small', live_migration_timeout=300)
self.create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 200 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_instance_overwrite_all(self):
create_instance_type('small', live_migration_timeout=300)
create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=100)
self.create_instance_type('small', live_migration_timeout=300)
self.create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=100)
assert 100 == instance.max_live_migrate_wait_in_secs
def test_live_migration_completion_timeout_overwrite_zero(self):
create_instance_type('small', live_migration_timeout=300)
create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "0"})
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=400)
self.create_instance_type('small', live_migration_timeout=300)
self.create_image('image_0',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "0"})
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0',
live_migration_timeout=400)
assert 300 == instance.max_live_migrate_wait_in_secs
create_instance_type('small_2', live_migration_timeout=0)
create_image('image_1',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = create_instance('test_instance_0',
'small_2',
'image_1',
'compute-0')
self.create_instance_type('small_2', live_migration_timeout=0)
self.create_image('image_1',
properties={nfvi_objects.IMAGE_PROPERTY.LIVE_MIGRATION_TIMEOUT: "200"})
instance = self.create_instance('test_instance_0',
'small_2',
'image_1',
'compute-0')
assert 200 == instance.max_live_migrate_wait_in_secs
# NOTE: End of tests from nova
@ -367,18 +349,18 @@ class TestInstance(object):
def test_live_migration_completion_timeout_out_of_range(self):
config.CONF = {'instance-configuration': {}}
create_instance_type('small', live_migration_timeout=1000)
create_image('image_0')
instance = create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
self.create_instance_type('small', live_migration_timeout=1000)
self.create_image('image_0')
instance = self.create_instance('test_instance_0',
'small',
'image_0',
'compute-0')
assert 800 == instance.max_live_migrate_wait_in_secs
create_instance_type('small_2', live_migration_timeout=10)
create_image('image_0')
instance = create_instance('test_instance_1',
'small_2',
'image_0',
'compute-0')
self.create_instance_type('small_2', live_migration_timeout=10)
self.create_image('image_0')
instance = self.create_instance('test_instance_1',
'small_2',
'image_0',
'compute-0')
assert 120 == instance.max_live_migrate_wait_in_secs

@ -3,17 +3,18 @@
#
# SPDX-License-Identifier: Apache-2.0
#
import uuid
import mock
import nose
import uuid
from nfv_vim import nfvi
from nfv_vim import objects
from nfv_vim.tables._image_table import ImageTable
from nfv_vim.tables._table import Table
from nfv_vim.directors._instance_director import InstanceDirector
import utils
from . import testcase
from . import utils
# Constants
_audit_interval = 330
@ -23,369 +24,375 @@ _rebuild_timeout = 300
_reboot_timeout = 300
_migrate_timeout = 600
# Globals
_tenant_table = Table()
_instance_table = Table()
_instance_type_table = Table()
_director = None
def create_instance(instance_type_name, instance_name, recovery_priority=None):
"""
Create an instance
"""
global _tenant_table
tenant_uuid = uuid.uuid4()
image_uuid = uuid.uuid4()
tenant = objects.Tenant(tenant_uuid, "%s_name" % tenant_uuid, '', True)
_tenant_table[tenant_uuid] = tenant
for instance_type in _instance_type_table.values():
if instance_type.name == instance_type_name:
instance_uuid = uuid.uuid4()
nfvi_instance = nfvi.objects.v1.Instance(
instance_uuid, instance_name, tenant_uuid,
admin_state=nfvi.objects.v1.INSTANCE_ADMIN_STATE.UNLOCKED,
oper_state=nfvi.objects.v1.INSTANCE_OPER_STATE.ENABLED,
avail_status=list(),
action=nfvi.objects.v1.INSTANCE_ACTION.NONE,
host_name='compute-0',
instance_type=utils.instance_type_to_flavor_dict(
instance_type),
image_uuid=image_uuid,
recovery_priority=recovery_priority)
return objects.Instance(nfvi_instance)
return None
@mock.patch('nfv_common.timers.timers_create_timer')
def instance_setup_func(timers_create_timer_mock):
"""
Setup for testing.
"""
global _instance_table, _instance_type_table, _director
instance_type_uuid = uuid.uuid4()
if 0 == len(_instance_type_table):
instance_type = objects.InstanceType(instance_type_uuid, 'small')
instance_type.update_details(vcpus=1, mem_mb=64, disk_gb=1, ephemeral_gb=0,
swap_gb=0, guest_services=None,
auto_recovery=True,
live_migration_timeout=800,
live_migration_max_downtime=500,
storage_type='local_image')
_instance_type_table[instance_type_uuid] = instance_type
_instance_table.clear()
if _director is None:
_director = InstanceDirector(
max_concurrent_recovering_instances=4,
max_concurrent_migrates_per_host=1,
max_concurrent_evacuates_per_host=1,
recovery_audit_interval=_audit_interval,
recovery_audit_cooldown=_audit_cooldown,
recovery_audit_batch_interval=2,
recovery_cooldown=_recovery_cooldown,
rebuild_timeout=_rebuild_timeout,
reboot_timeout=_reboot_timeout,
migrate_timeout=_migrate_timeout,
single_hypervisor=False,
recovery_threshold=250,
max_throttled_recovering_instances=2
)
def instance_teardown_func():
"""
Cleanup testing setup.
"""
pass
@nose.with_setup(instance_setup_func, instance_teardown_func)
@mock.patch('nfv_vim.tables.tables_get_tenant_table')
@mock.patch('nfv_vim.tables.tables_get_instance_type_table')
@mock.patch('nfv_vim.tables.tables_get_instance_table')
def test_instance_director_recovery_list(
tables_get_instance_table_mock, tables_get_instance_type_table_mock,
tables_get_tenant_table_mock):
"""
Test the instance director recovery list logic
"""
global _tenant_table, _instance_table, _instance_type_table, _director
tables_get_tenant_table_mock.return_value = _tenant_table
tables_get_instance_table_mock.return_value = _instance_table
tables_get_instance_type_table_mock.return_value = _instance_type_table
instance_1 = create_instance('small', 'instance_1')
_instance_table[instance_1.uuid] = instance_1
# Validate that the Instance Director recovery_list creation
# -- with no instances in the failed state, verify that the list is
# empty and the normal audit interval is returned
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _audit_interval
assert 0 == len(instance_recovery_list)
instance_1._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
# -- with one instance in the failed state, but elapsed time is less
# than the recovery cooldown, verify that the list is empty, but
# the audit interval is set to the recovery cooldown period
instance_1._elapsed_time_in_state = _recovery_cooldown - 1
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 0 == len(instance_recovery_list)
# -- with one instance in the failed state, but elapsed time is greater
# than or equal to the recovery cooldown, verify that the list contains
# one instance and the audit interval is set to the recovery cooldown
# period
instance_1._elapsed_time_in_state = _recovery_cooldown
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 1 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_1.name
@nose.with_setup(instance_setup_func, instance_teardown_func)
@mock.patch('nfv_vim.tables.tables_get_tenant_table')
@mock.patch('nfv_vim.tables.tables_get_instance_type_table')
@mock.patch('nfv_vim.tables.tables_get_instance_table')
def test_instance_director_recovery_list_order(
tables_get_instance_table_mock, tables_get_instance_type_table_mock,
tables_get_tenant_table_mock):
"""
Test the instance director recovery list ordering
"""
global _tenant_table, _instance_table, _instance_type_table, _director
tables_get_tenant_table_mock.return_value = _tenant_table
tables_get_instance_table_mock.return_value = _instance_table
tables_get_instance_type_table_mock.return_value = _instance_type_table
instance_1 = create_instance('small', 'instance_1')
instance_1._elapsed_time_in_state = _recovery_cooldown
instance_1._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_1._nfvi_instance['instance_type']['vcpus'] = 1
instance_1._nfvi_instance['instance_type']['ram'] = 32
instance_1._nfvi_instance['instance_type']['disk'] = 2
instance_1._nfvi_instance['instance_type']['swap'] = 0
_instance_table[instance_1.uuid] = instance_1
# Validate the Instance Director recovery_list order
# -- with one instance in the failed state
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 1 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_1.name
instance_2 = create_instance('small', 'instance_2')
instance_2._elapsed_time_in_state = _recovery_cooldown
instance_2._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_2._nfvi_instance['instance_type']['vcpus'] = 2
instance_2._nfvi_instance['instance_type']['ram'] = 32
instance_2._nfvi_instance['instance_type']['disk'] = 1
instance_2._nfvi_instance['instance_type']['swap'] = 0
_instance_table[instance_2.uuid] = instance_2
# -- with two instances in the failed state
# vcpus takes precedence over disk_gb
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 2 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_2.name
assert instance_recovery_list[1].name == instance_1.name
instance_3 = create_instance('small', 'instance_3', recovery_priority=5)
instance_3._elapsed_time_in_state = _recovery_cooldown
instance_3._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_3._nfvi_instance['instance_type']['vcpus'] = 1
instance_3._nfvi_instance['instance_type']['ram'] = 32
instance_3._nfvi_instance['instance_type']['disk'] = 0
instance_3._nfvi_instance['instance_type']['swap'] = 0
_instance_table[instance_3.uuid] = instance_3
# -- with three instances in the failed state
# recovery_priority takes precedence over instance size
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 3 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_3.name
assert instance_recovery_list[1].name == instance_2.name
assert instance_recovery_list[2].name == instance_1.name
instance_4 = create_instance('small', 'instance_4', recovery_priority=1)
instance_4._elapsed_time_in_state = _recovery_cooldown
instance_4._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_4._nfvi_instance['instance_type']['vcpus'] = 1
instance_4._nfvi_instance['instance_type']['ram'] = 32
instance_4._nfvi_instance['instance_type']['disk'] = 0
instance_4._nfvi_instance['instance_type']['swap'] = 0
_instance_table[instance_4.uuid] = instance_4
# -- with four instances in the failed state
# recovery_priority sorts instances
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 4 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_4.name
assert instance_recovery_list[1].name == instance_3.name
assert instance_recovery_list[2].name == instance_2.name
assert instance_recovery_list[3].name == instance_1.name
instance_5 = create_instance('small', 'instance_5', recovery_priority=10)
instance_5._elapsed_time_in_state = _recovery_cooldown
instance_5._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_5._nfvi_instance['instance_type']['vcpus'] = 2
instance_5._nfvi_instance['instance_type']['ram'] = 32
instance_5._nfvi_instance['instance_type']['disk'] = 0
instance_5._nfvi_instance['instance_type']['swap'] = 0
_instance_table[instance_5.uuid] = instance_5
# -- with five instances in the failed state
# no recovery_priority treated the same as priority 10
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = _director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 5 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_4.name
assert instance_recovery_list[1].name == instance_3.name
assert instance_recovery_list[2].name == instance_2.name
assert instance_recovery_list[3].name == instance_5.name
assert instance_recovery_list[4].name == instance_1.name
@nose.with_setup(instance_setup_func, instance_teardown_func)
@mock.patch('nfv_vim.tables.tables_get_tenant_table')
@mock.patch('nfv_vim.tables.tables_get_instance_type_table')
@mock.patch('nfv_vim.tables.tables_get_instance_table')
@mock.patch('nfv_vim.dor.system_is_stabilized')
@mock.patch('nfv_vim.dor.dor_is_complete')
def test_instance_director_recover_instance(
dor_is_complete_mock, system_is_stabilized_mock,
tables_get_instance_table_mock, tables_get_instance_type_table_mock,
tables_get_tenant_table_mock):
"""
Test the instance director recover instance logic
"""
global _instance_table, _instance_type_table, _director
tables_get_tenant_table_mock.return_value = _tenant_table
tables_get_instance_table_mock.return_value = _instance_table
tables_get_instance_type_table_mock.return_value = _instance_type_table
system_is_stabilized_mock.return_value = True
dor_is_complete_mock.return_value = True
instance_1 = create_instance('small', 'instance_1')
instance_1.fail = mock.Mock()
instance_1.do_action = mock.Mock()
instance_1._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
_director._is_host_enabled = mock.Mock(return_value=True)
_director._is_hypervisor_enabled = mock.Mock(return_value=True)
_director.upgrade_inprogress = mock.Mock(return_value=False)
# Set_A
# -- the first attempt to recover an instance that is failed on an
# enabled host, verify a reboot is attempted
_director.instance_recovered(instance_1)
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBOOT,
initiated_by='director')
# -- a subsequent attempt to recover an instance that is failed on an
# enabled host and the instance has an image to rebuild from, verify
# a rebuild is attempted
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBUILD,
initiated_by='director')
# Set_B
# -- the first attempt to recover an instance that is failed on an
# enabled host, verify a reboot is attempted
original_image_uuid = instance_1._nfvi_instance.image_uuid
instance_1._nfvi_instance.image_uuid = None
_director.instance_recovered(instance_1)
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBOOT,
initiated_by='director')
# -- a subsequent attempt to recover an instance that is failed on an
# enabled host and the instance does not have an image to rebuild from,
# verify a reboot is attempted
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBOOT,
initiated_by='director')
instance_1._nfvi_instance.image_uuid = original_image_uuid
# Set_C
# -- instance is rebuilding or evacuating and the instance has an image
# to rebuild from, verify that a rebuild is attempted
_director.instance_recovered(instance_1)
instance_1._nfvi_instance.action = nfvi.objects.v1.INSTANCE_ACTION.REBUILDING
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBUILD,
initiated_by='director')
# Set_D
# -- instance is rebuilding or evacuating and the instance does not have an
# image to rebuild from, verify that a reboot is attempted
original_image_uuid = instance_1._nfvi_instance.image_uuid
instance_1._nfvi_instance.image_uuid = None
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBOOT,
initiated_by='director')
instance_1._nfvi_instance.image_uuid = original_image_uuid
instance_1._nfvi_instance.action = nfvi.objects.v1.INSTANCE_ACTION.NONE
# Set_E
# -- instance is migrating and the instance has an image to rebuild from,
# verify that a rebuild is attempted
_director.instance_recovered(instance_1)
instance_1._nfvi_instance.action = nfvi.objects.v1.INSTANCE_ACTION.MIGRATING
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBUILD,
initiated_by='director')
# Set_F
# -- instance is migrating and the instance does not have an image to
# rebuild from, verify that a reboot is attempted
original_image_uuid = instance_1._nfvi_instance.image_uuid
instance_1._nfvi_instance.image_uuid = None
_director.recover_instance(instance_1)
instance_1.do_action.assert_called_with(objects.INSTANCE_ACTION_TYPE.REBOOT,
initiated_by='director')
instance_1._nfvi_instance.image_uuid = original_image_uuid
instance_1._nfvi_instance.action = nfvi.objects.v1.INSTANCE_ACTION.NONE
def fake_timer(a, b, c, d):
return 1234
class TestInstanceDirector(testcase.NFVTestCase):
_image_table = ImageTable()
_instance_table = Table()
_instance_type_table = Table()
_tenant_table = Table()
# Don't attempt to write to the database while unit testing
_image_table.persist = False
_instance_table.persist = False
_instance_type_table.persist = False
_tenant_table.persist = False
_director = None
def setUp(self):
super(TestInstanceDirector, self).setUp()
self.instance_setup_func()
def tearDown(self):
super(TestInstanceDirector, self).tearDown()
self._tenant_table.clear()
self._instance_table.clear()
self._instance_type_table.clear()
self._director = None
def create_instance(self, instance_type_name, instance_name, recovery_priority=None):
"""
Create an instance
"""
tenant_uuid = uuid.uuid4()
image_uuid = uuid.uuid4()
tenant = objects.Tenant(tenant_uuid, "%s_name" % tenant_uuid, '', True)
self._tenant_table[tenant_uuid] = tenant
for instance_type in self._instance_type_table.values():
if instance_type.name == instance_type_name:
instance_uuid = uuid.uuid4()
nfvi_instance = nfvi.objects.v1.Instance(
instance_uuid, instance_name, tenant_uuid,
admin_state=nfvi.objects.v1.INSTANCE_ADMIN_STATE.UNLOCKED,
oper_state=nfvi.objects.v1.INSTANCE_OPER_STATE.ENABLED,
avail_status=list(),
action=nfvi.objects.v1.INSTANCE_ACTION.NONE,
host_name='compute-0',
instance_type=utils.instance_type_to_flavor_dict(
instance_type),
image_uuid=image_uuid,
recovery_priority=recovery_priority)
return objects.Instance(nfvi_instance)
return None
@mock.patch('nfv_common.timers.timers_create_timer', fake_timer)
def instance_setup_func(self):
"""
Setup for testing.
"""
instance_type_uuid = uuid.uuid4()
if 0 == len(self._instance_type_table):
instance_type = objects.InstanceType(instance_type_uuid, 'small')
instance_type.update_details(vcpus=1, mem_mb=64, disk_gb=1, ephemeral_gb=0,
swap_gb=0, guest_services=None,
auto_recovery=True,
live_migration_timeout=800,
live_migration_max_downtime=500,
storage_type='local_image')
self._instance_type_table[instance_type_uuid] = instance_type
self._instance_table.clear()
if self._director is None:
self._director = InstanceDirector(
max_concurrent_recovering_instances=4,
max_concurrent_migrates_per_host=1,
max_concurrent_evacuates_per_host=1,
recovery_audit_interval=_audit_interval,
recovery_audit_cooldown=_audit_cooldown,
recovery_audit_batch_interval=2,
recovery_cooldown=_recovery_cooldown,
rebuild_timeout=_rebuild_timeout,
reboot_timeout=_reboot_timeout,
migrate_timeout=_migrate_timeout,
single_hypervisor=False,
recovery_threshold=250,
max_throttled_recovering_instances=2
)
@mock.patch('nfv_vim.tables.tables_get_tenant_table')
@mock.patch('nfv_vim.tables.tables_get_instance_type_table')
@mock.patch('nfv_vim.tables.tables_get_instance_table')
def test_instance_director_recovery_list(self,
tables_get_instance_table_mock,
tables_get_instance_type_table_mock,
tables_get_tenant_table_mock):
"""
Test the instance director recovery list logic
"""
tables_get_tenant_table_mock.return_value = self._tenant_table
tables_get_instance_table_mock.return_value = self._instance_table
tables_get_instance_type_table_mock.return_value = self._instance_type_table
instance_1 = self.create_instance('small', 'instance_1')
self._instance_table[instance_1.uuid] = instance_1
# Validate that the Instance Director recovery_list creation
# -- with no instances in the failed state, verify that the list is
# empty and the normal audit interval is returned
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = self._director._get_instance_recovery_list()
assert next_audit_interval == _audit_interval
assert 0 == len(instance_recovery_list)
instance_1._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
# -- with one instance in the failed state, but elapsed time is less
# than the recovery cooldown, verify that the list is empty, but
# the audit interval is set to the recovery cooldown period
instance_1._elapsed_time_in_state = _recovery_cooldown - 1
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = self._director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 0 == len(instance_recovery_list)
# -- with one instance in the failed state, but elapsed time is greater
# than or equal to the recovery cooldown, verify that the list contains
# one instance and the audit interval is set to the recovery cooldown
# period
instance_1._elapsed_time_in_state = _recovery_cooldown
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = self._director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 1 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_1.name
@mock.patch('nfv_vim.tables.tables_get_tenant_table')
@mock.patch('nfv_vim.tables.tables_get_instance_type_table')
@mock.patch('nfv_vim.tables.tables_get_instance_table')
def test_instance_director_recovery_list_order(self,
tables_get_instance_table_mock,
tables_get_instance_type_table_mock,
tables_get_tenant_table_mock):
"""
Test the instance director recovery list ordering
"""
tables_get_tenant_table_mock.return_value = self._tenant_table
tables_get_instance_table_mock.return_value = self._instance_table
tables_get_instance_type_table_mock.return_value = self._instance_type_table
instance_1 = self.create_instance('small', 'instance_1')
instance_1._elapsed_time_in_state = _recovery_cooldown
instance_1._nfvi_instance.avail_status.append(
nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED)
instance_1._nfvi_instance['instance_type']['vcpus'] = 1
instance_1._nfvi_instance['instance_type']['ram'] = 32
instance_1._nfvi_instance['instance_type']['disk'] = 2
instance_1._nfvi_instance['instance_type']['swap'] = 0
self._instance_table[instance_1.uuid] = instance_1
# Validate the Instance Director recovery_list order
# -- with one instance in the failed state
(next_audit_interval, instance_recovery_list, instance_failed_list,
instance_rebuilding_list, instance_migrating_list,
instance_rebooting_list) = self._director._get_instance_recovery_list()
assert next_audit_interval == _recovery_cooldown
assert 1 == len(instance_recovery_list)
assert instance_recovery_list[0].name == instance_1.name
instance_2 = self.create_instance('small', '<