openstack/deb-manila
Code Issues Proposed changes

Support for oversubscription in thin provisioning

Browse Source 
This patch adds support for oversubscription in thin provisioning
in Manila. This is similar to the same feature in Cinder.

Change-Id: I3e45ad23a83591bf16adff7f21c3ce19325e7e91
implements: blueprint oversubscription-in-thin-provisioning
This commit is contained in:
Xing Yang 2015-06-15 23:24:46 -04:00
parent dbf00f2e32
commit 811f496acd
10 changed files with 526 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
manila/scheduler/filters/capacity_filter.py
View File

@ -1,5 +1,6 @@
# Copyright (c) 2012 Intel # Copyright (c) 2012 Intel
# Copyright (c) 2012 OpenStack, LLC. # Copyright (c) 2012 OpenStack, LLC.
# Copyright (c) 2015 EMC Corporation
# #
# All Rights Reserved. # All Rights Reserved.
# #
@ -41,19 +42,79 @@ class CapacityFilter(filters.BaseHostFilter):
return False return False
free_space = host_state.free_capacity_gb free_space = host_state.free_capacity_gb
if free_space == 'infinite' or free_space == 'unknown': total_space = host_state.total_capacity_gb
reserved = float(host_state.reserved_percentage) / 100
if free_space in ('infinite', 'unknown'):
# NOTE(zhiteng) for those back-ends cannot report actual # NOTE(zhiteng) for those back-ends cannot report actual
# available capacity, we assume it is able to serve the # available capacity, we assume it is able to serve the
# request. Even if it was not, the retry mechanism is # request. Even if it was not, the retry mechanism is
# able to handle the failure by rescheduling # able to handle the failure by rescheduling
return True return True
reserved = float(host_state.reserved_percentage) / 100 elif total_space in ('infinite', 'unknown'):
free = math.floor(free_space * (1 - reserved)) # NOTE(xyang): If total_space is 'infinite' or 'unknown' and
# reserved is 0, we assume the back-ends can serve the request.
# If total_space is 'infinite' or 'unknown' and reserved
# is not 0, we cannot calculate the reserved space.
# float(total_space) will throw an exception. total*reserved
# also won't work. So the back-ends cannot serve the request.
return reserved == 0
total = float(total_space)
if total <= 0:
LOG.warning(_LW("Insufficient free space for share creation. "
"Total capacity is %(total).2f on host %(host)s."),
{"total": total,
"host": host_state.host})
return False
# NOTE(xyang): Calculate how much free space is left after taking
# into account the reserved space.
free = math.floor(free_space - total * reserved)
msg_args = {"host": host_state.host,
"requested": share_size,
"available": free}
LOG.debug("Space information for share creation "
"on host %(host)s (requested / avail): "
"%(requested)s/%(available)s", msg_args)
# NOTE(xyang): Only evaluate using max_over_subscription_ratio
# if thin_provisioning_support is True. Check if the ratio of
# provisioned capacity over total capacity would exceed
# subscription ratio.
# If max_over_subscription_ratio = 1, the provisioned_ratio
# should still be limited by the max_over_subscription_ratio;
# otherwise, it could result in infinite provisioning.
if (host_state.thin_provisioning_support and
host_state.max_over_subscription_ratio >= 1):
provisioned_ratio = ((host_state.provisioned_capacity_gb +
share_size) / total)
if provisioned_ratio > host_state.max_over_subscription_ratio:
LOG.warning(_LW(
"Insufficient free space for thin provisioning. "
"The ratio of provisioned capacity over total capacity "
"%(provisioned_ratio).2f would exceed the maximum over "
"subscription ratio %(oversub_ratio).2f on host "
"%(host)s."),
{"provisioned_ratio": provisioned_ratio,
"oversub_ratio": host_state.max_over_subscription_ratio,
"host": host_state.host})
return False
else:
# NOTE(xyang): Adjust free_virtual calculation based on
# free and max_over_subscription_ratio.
adjusted_free_virtual = (
free * host_state.max_over_subscription_ratio)
return adjusted_free_virtual >= share_size
elif host_state.thin_provisioning_support:
LOG.error(_LE("Invalid max_over_subscription_ratio: %(ratio)s. "
"Valid value should be >= 1."),
{"ratio": host_state.max_over_subscription_ratio})
return False
if free < share_size: if free < share_size:
LOG.warning(_LW("Insufficient free space for share creation " LOG.warning(_LW("Insufficient free space for share creation "
"(requested / avail): " "on host %(host)s (requested / avail): "
"%(requested)s/%(available)s"), "%(requested)s/%(available)s"), msg_args)
{'requested': share_size, return False
'available': free})
return free >= share_size return True

26
manila/scheduler/host_manager.py
View File

@ -1,6 +1,7 @@
# Copyright (c) 2011 OpenStack, LLC. # Copyright (c) 2011 OpenStack, LLC.
# Copyright (c) 2015 Rushil Chugh # Copyright (c) 2015 Rushil Chugh
# Copyright (c) 2015 Clinton Knight # Copyright (c) 2015 Clinton Knight
# Copyright (c) 2015 EMC Corporation
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -53,6 +54,7 @@ host_manager_opts = [
CONF = cfg.CONF CONF = cfg.CONF
CONF.register_opts(host_manager_opts) CONF.register_opts(host_manager_opts)
CONF.import_opt('max_over_subscription_ratio', 'manila.share.driver')
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
@ -108,6 +110,15 @@ class HostState(object):
self.total_capacity_gb = 0 self.total_capacity_gb = 0
self.free_capacity_gb = None self.free_capacity_gb = None
self.reserved_percentage = 0 self.reserved_percentage = 0
self.allocated_capacity_gb = 0
# NOTE(xyang): The apparent allocated space indicating how much
# capacity has been provisioned. This could be the sum of sizes
# of all shares on a backend, which could be greater than or
# equal to the allocated_capacity_gb.
self.provisioned_capacity_gb = 0
self.max_over_subscription_ratio = 1.0
self.thin_provisioning_support = False
self.thick_provisioning_support = False
self.driver_handles_share_servers = False self.driver_handles_share_servers = False
# PoolState for all pools # PoolState for all pools
@ -313,6 +324,21 @@ class PoolState(HostState):
'allocated_capacity_gb', 0) 'allocated_capacity_gb', 0)
self.QoS_support = capability.get('QoS_support', False) self.QoS_support = capability.get('QoS_support', False)
self.reserved_percentage = capability['reserved_percentage'] self.reserved_percentage = capability['reserved_percentage']
# NOTE(xyang): provisioned_capacity_gb is the apparent total
# capacity of all the shares created on a backend, which is
# greater than or equal to allocated_capacity_gb, which is the
# apparent total capacity of all the shares created on a backend
# in Manila. Using allocated_capacity_gb as the default of
# provisioned_capacity_gb if it is not set.
self.provisioned_capacity_gb = capability.get(
'provisioned_capacity_gb', self.allocated_capacity_gb)
self.max_over_subscription_ratio = capability.get(
'max_over_subscription_ratio',
CONF.max_over_subscription_ratio)
self.thin_provisioning_support = capability.get(
'thin_provisioning_support', False)
self.thick_provisioning_support = capability.get(
'thick_provisioning_support', False)
def update_pools(self, capability): def update_pools(self, capability):
# Do nothing, since we don't have pools within pool, yet # Do nothing, since we don't have pools within pool, yet

25
manila/scheduler/weights/capacity.py
View File

@ -1,4 +1,6 @@
# Copyright (c) 2012 OpenStack, LLC. # Copyright (c) 2012 OpenStack, LLC.
# Copyright (c) 2015 EMC Corporation
#
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,7 +15,12 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
""" """
Capacity Weigher. Weigh hosts by their available capacity. Capacity Weigher. Weigh hosts by their virtual or actual free capacity.
For thin provisioning, weigh hosts by their virtual free capacity calculated
by the total capacity multiplied by the max over subscription ratio and
subtracting the provisioned capacity; Otherwise, weigh hosts by their actual
free capacity, taking into account the reserved space.
The default is to spread shares across all hosts evenly. If you prefer The default is to spread shares across all hosts evenly. If you prefer
stacking, you can set the 'capacity_weight_multiplier' option to a negative stacking, you can set the 'capacity_weight_multiplier' option to a negative
@ -47,10 +54,22 @@ class CapacityWeigher(weights.BaseHostWeigher):
"""Higher weights win. We want spreading to be the default.""" """Higher weights win. We want spreading to be the default."""
reserved = float(host_state.reserved_percentage) / 100 reserved = float(host_state.reserved_percentage) / 100
free_space = host_state.free_capacity_gb free_space = host_state.free_capacity_gb
if free_space == 'infinite' or free_space == 'unknown': total_space = host_state.total_capacity_gb
if {'unknown', 'infinite'}.intersection({total_space, free_space}):
# (zhiteng) 'infinite' and 'unknown' are treated the same # (zhiteng) 'infinite' and 'unknown' are treated the same
# here, for sorting purpose. # here, for sorting purpose.
free = float('inf') free = float('inf')
else: else:
free = math.floor(host_state.free_capacity_gb * (1 - reserved)) total = float(total_space)
if host_state.thin_provisioning_support:
# NOTE(xyang): Calculate virtual free capacity for thin
# provisioning.
free = math.floor(
total * host_state.max_over_subscription_ratio -
host_state.provisioned_capacity_gb -
total * reserved)
else:
# NOTE(xyang): Calculate how much free space is left after
# taking into account the reserved space.
free = math.floor(free_space - total * reserved)
return free return free

16
manila/share/driver.py
View File

@ -61,6 +61,17 @@ share_opts = [
"is able to handle share servers and it is desired mode else set " "is able to handle share servers and it is desired mode else set "
"False. It is set to None by default to make this choice " "False. It is set to None by default to make this choice "
"intentional."), "intentional."),
cfg.FloatOpt(
'max_over_subscription_ratio',
default=20.0,
help='Float representation of the over subscription ratio '
'when thin provisioning is involved. Default ratio is '
'20.0, meaning provisioned capacity can be 20 times '
'the total physical capacity. If the ratio is 10.5, it '
'means provisioned capacity can be 10.5 times the '
'total physical capacity. A ratio of 1.0 means '
'provisioned capacity cannot exceed the total physical '
'capacity. A ratio lower than 1.0 is invalid.'),
] ]
ssh_opts = [ ssh_opts = [
@ -287,6 +298,11 @@ class ShareDriver(object):
def check_for_setup_error(self): def check_for_setup_error(self):
"""Check for setup error.""" """Check for setup error."""
max_ratio = self.configuration.safe_get('max_over_subscription_ratio')
if max_ratio < 1.0:
msg = (_("Invalid max_over_subscription_ratio '%s'. "
"Valid value should be >= 1.0.") % max_ratio)
raise exception.InvalidParameterValue(err=msg)
def do_setup(self, context): def do_setup(self, context):
"""Any initialization the share driver does while starting.""" """Any initialization the share driver does while starting."""

3
manila/tests/fake_driver.py
View File

@ -69,9 +69,6 @@ class FakeShareDriver(driver.ShareDriver):
def deny_access(self, context, share, access, share_server=None): def deny_access(self, context, share, access, share_server=None):
pass pass
def check_for_setup_error(self):
pass
def get_share_stats(self, refresh=False): def get_share_stats(self, refresh=False):
return None return None

81
manila/tests/scheduler/fakes.py
View File

@ -35,14 +35,26 @@ SERVICE_STATES_NO_POOLS = {
'host1': dict(share_backend_name='AAA', 'host1': dict(share_backend_name='AAA',
total_capacity_gb=512, free_capacity_gb=200, total_capacity_gb=512, free_capacity_gb=200,
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
provisioned_capacity_gb=312,
max_over_subscription_ratio=1.0,
thin_provisioning_support=False,
thick_provisioning_support=True,
driver_handles_share_servers=False), driver_handles_share_servers=False),
'host2@back1': dict(share_backend_name='BBB', 'host2@back1': dict(share_backend_name='BBB',
total_capacity_gb=256, free_capacity_gb=100, total_capacity_gb=256, free_capacity_gb=100,
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
provisioned_capacity_gb=400,
max_over_subscription_ratio=2.0,
thin_provisioning_support=True,
thick_provisioning_support=False,
driver_handles_share_servers=False), driver_handles_share_servers=False),
'host2@back2': dict(share_backend_name='CCC', 'host2@back2': dict(share_backend_name='CCC',
total_capacity_gb=10000, free_capacity_gb=700, total_capacity_gb=10000, free_capacity_gb=700,
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
provisioned_capacity_gb=50000,
max_over_subscription_ratio=20.0,
thin_provisioning_support=True,
thick_provisioning_support=True,
driver_handles_share_servers=False), driver_handles_share_servers=False),
} }
@ -67,43 +79,71 @@ SHARE_SERVICE_STATES_WITH_POOLS = {
pools=[dict(pool_name='pool1', pools=[dict(pool_name='pool1',
total_capacity_gb=51, total_capacity_gb=51,
free_capacity_gb=41, free_capacity_gb=41,
reserved_percentage=0)]), reserved_percentage=0,
provisioned_capacity_gb=10,
max_over_subscription_ratio=1.0,
thin_provisioning_support=False,
thick_provisioning_support=True)]),
'host2@BBB': dict(share_backend_name='BBB', 'host2@BBB': dict(share_backend_name='BBB',
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
driver_handles_share_servers=False, driver_handles_share_servers=False,
pools=[dict(pool_name='pool2', pools=[dict(pool_name='pool2',
total_capacity_gb=52, total_capacity_gb=52,
free_capacity_gb=42, free_capacity_gb=42,
reserved_percentage=0)]), reserved_percentage=0,
provisioned_capacity_gb=60,
max_over_subscription_ratio=2.0,
thin_provisioning_support=True,
thick_provisioning_support=False)]),
'host3@CCC': dict(share_backend_name='CCC', 'host3@CCC': dict(share_backend_name='CCC',
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
driver_handles_share_servers=False, driver_handles_share_servers=False,
pools=[dict(pool_name='pool3', pools=[dict(pool_name='pool3',
total_capacity_gb=53, total_capacity_gb=53,
free_capacity_gb=43, free_capacity_gb=43,
reserved_percentage=0)]), reserved_percentage=0,
provisioned_capacity_gb=100,
max_over_subscription_ratio=20.0,
thin_provisioning_support=True,
thick_provisioning_support=True)]),
'host4@DDD': dict(share_backend_name='DDD', 'host4@DDD': dict(share_backend_name='DDD',
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
driver_handles_share_servers=False, driver_handles_share_servers=False,
pools=[dict(pool_name='pool4a', pools=[dict(pool_name='pool4a',
total_capacity_gb=541, total_capacity_gb=541,
free_capacity_gb=441, free_capacity_gb=441,
reserved_percentage=0), reserved_percentage=0,
provisioned_capacity_gb=800,
max_over_subscription_ratio=2.0,
thin_provisioning_support=True,
thick_provisioning_support=False),
dict(pool_name='pool4b', dict(pool_name='pool4b',
total_capacity_gb=542, total_capacity_gb=542,
free_capacity_gb=442, free_capacity_gb=442,
reserved_percentage=0)]), reserved_percentage=0,
provisioned_capacity_gb=2000,
max_over_subscription_ratio=10.0,
thin_provisioning_support=True,
thick_provisioning_support=False)]),
'host5@EEE': dict(share_backend_name='EEE', 'host5@EEE': dict(share_backend_name='EEE',
timestamp=None, reserved_percentage=0, timestamp=None, reserved_percentage=0,
driver_handles_share_servers=False, driver_handles_share_servers=False,
pools=[dict(pool_name='pool5a', pools=[dict(pool_name='pool5a',
total_capacity_gb=551, total_capacity_gb=551,
free_capacity_gb=451, free_capacity_gb=451,
reserved_percentage=0), reserved_percentage=0,
provisioned_capacity_gb=100,
max_over_subscription_ratio=1.0,
thin_provisioning_support=False,
thick_provisioning_support=True),
dict(pool_name='pool5b', dict(pool_name='pool5b',
total_capacity_gb=552, total_capacity_gb=552,
free_capacity_gb=452, free_capacity_gb=452,
reserved_percentage=0)]), reserved_percentage=0,
provisioned_capacity_gb=100,
max_over_subscription_ratio=1.0,
thin_provisioning_support=False,
thick_provisioning_support=True)]),
} }
@ -120,22 +160,47 @@ class FakeHostManager(host_manager.HostManager):
self.service_states = { self.service_states = {
'host1': {'total_capacity_gb': 1024, 'host1': {'total_capacity_gb': 1024,
'free_capacity_gb': 1024, 'free_capacity_gb': 1024,
'allocated_capacity_gb': 0,
'provisioned_capacity_gb': 0,
'max_over_subscription_ratio': 1.0,
'thin_provisioning_support': False,
'thick_provisioning_support': True,
'reserved_percentage': 10, 'reserved_percentage': 10,
'timestamp': None}, 'timestamp': None},
'host2': {'total_capacity_gb': 2048, 'host2': {'total_capacity_gb': 2048,
'free_capacity_gb': 300, 'free_capacity_gb': 300,
'allocated_capacity_gb': 1748,
'provisioned_capacity_gb': 1748,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'reserved_percentage': 10, 'reserved_percentage': 10,
'timestamp': None}, 'timestamp': None},
'host3': {'total_capacity_gb': 512, 'host3': {'total_capacity_gb': 512,
'free_capacity_gb': 512, 'free_capacity_gb': 256,
'allocated_capacity_gb': 256,
'provisioned_capacity_gb': 256,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': False,
'thick_provisioning_support': True,
'reserved_percentage': 0, 'reserved_percentage': 0,
'timestamp': None}, 'timestamp': None},
'host4': {'total_capacity_gb': 2048, 'host4': {'total_capacity_gb': 2048,
'free_capacity_gb': 200, 'free_capacity_gb': 200,
'allocated_capacity_gb': 1848,
'provisioned_capacity_gb': 1848,
'max_over_subscription_ratio': 1.0,
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'reserved_percentage': 5, 'reserved_percentage': 5,
'timestamp': None}, 'timestamp': None},
'host5': {'total_capacity_gb': 2048, 'host5': {'total_capacity_gb': 2048,
'free_capacity_gb': 500, 'free_capacity_gb': 500,
'allocated_capacity_gb': 1548,
'provisioned_capacity_gb': 1548,
'max_over_subscription_ratio': 1.5,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'reserved_percentage': 5, 'reserved_percentage': 5,
'timestamp': None}, 'timestamp': None},
} }

118
manila/tests/scheduler/test_capacity_weigher.py
View File

@ -38,7 +38,7 @@ class CapacityWeigherTestCase(test.TestCase):
def _get_weighed_host(self, hosts, weight_properties=None): def _get_weighed_host(self, hosts, weight_properties=None):
if weight_properties is None: if weight_properties is None:
weight_properties = {} weight_properties = {'size': 1}
return self.weight_handler.get_weighed_objects( return self.weight_handler.get_weighed_objects(
[capacity.CapacityWeigher], [capacity.CapacityWeigher],
hosts, hosts,
@ -54,46 +54,120 @@ class CapacityWeigherTestCase(test.TestCase):
ctxt, CONF.share_topic) ctxt, CONF.share_topic)
return host_states return host_states
# NOTE(xyang): If thin_provisioning_support = True and
# max_over_subscription_ratio >= 1, use the following formula:
# free = math.floor(total * host_state.max_over_subscription_ratio
# - host_state.provisioned_capacity_gb
# - total * reserved)
# Otherwise, use the following formula:
# free = math.floor(free_space - total * reserved)
def test_default_of_spreading_first(self): def test_default_of_spreading_first(self):
hostinfo_list = self._get_all_hosts() hostinfo_list = self._get_all_hosts()
# host1: free_capacity_gb=1024, free=1024*(1-0.1) # host1: thin_provisioning_support = False
# host2: free_capacity_gb=300, free=300*(1-0.1) # free_capacity_gb = 1024
# host3: free_capacity_gb=512, free=512 # free = math.floor(1024 - 1024 * 0.1) = 921.0
# host4: free_capacity_gb=200, free=200*(1-0.05) # weight = 0.40
# host2: thin_provisioning_support = True
# max_over_subscription_ratio = 2.0
# free_capacity_gb = 300
# free = math.floor(2048 * 2.0 - 1748 - 2048 * 0.1)=2143.0
# weight = 1.0
# host3: thin_provisioning_support = False
# free_capacity_gb = 512
# free = math.floor(256 - 512 * 0)=256.0
# weight = 0.08
# host4: thin_provisioning_support = True
# max_over_subscription_ratio = 1.0
# free_capacity_gb = 200
# free = math.floor(2048 * 1.0 - 1848 - 2048 * 0.05) = 97.0
# weight = 0.0
# host5: thin_provisioning_support = True
# max_over_subscription_ratio = 1.5
# free_capacity_gb = 500
# free = math.floor(2048 * 1.5 - 1548 - 2048 * 0.05) = 1421.0
# weight = 0.65
# so, host1 should win: # so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list) weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 1.0) self.assertEqual(1.0, weighed_host.weight)
self.assertEqual( self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host)) 'host2', utils.extract_host(weighed_host.obj.host))
def test_capacity_weight_multiplier1(self): def test_capacity_weight_multiplier_negative_1(self):
self.flags(capacity_weight_multiplier=-1.0) self.flags(capacity_weight_multiplier=-1.0)
hostinfo_list = self._get_all_hosts() hostinfo_list = self._get_all_hosts()
# host1: free_capacity_gb=1024, free=-1024*(1-0.1) # host1: thin_provisioning_support = False
# host2: free_capacity_gb=300, free=-300*(1-0.1) # free_capacity_gb = 1024
# host3: free_capacity_gb=512, free=-512 # free = math.floor(1024 - 1024 * 0.1) = 921.0
# host4: free_capacity_gb=200, free=-200*(1-0.05) # free * (-1) = -921.0
# weight = -0.40
# host2: thin_provisioning_support = True
# max_over_subscription_ratio = 2.0
# free_capacity_gb = 300
# free = math.floor(2048 * 2.0-1748-2048 * 0.1) = 2143.0
# free * (-1) = -2143.0
# weight = -1.0
# host3: thin_provisioning_support = False
# free_capacity_gb = 512
# free = math.floor(256 - 512 * 0) = 256.0
# free * (-1) = -256.0
# weight = -0.08
# host4: thin_provisioning_support = True
# max_over_subscription_ratio = 1.0
# free_capacity_gb = 200
# free = math.floor(2048 * 1.0 - 1848 - 2048 * 0.05) = 97.0
# free * (-1) = -97.0
# weight = 0.0
# host5: thin_provisioning_support = True
# max_over_subscription_ratio = 1.5
# free_capacity_gb = 500
# free = math.floor(2048 * 1.5 - 1548 - 2048 * 0.05) = 1421.0
# free * (-1) = -1421.0
# weight = -0.65
# so, host4 should win: # so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list) weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 0.0) self.assertEqual(0.0, weighed_host.weight)
self.assertEqual( self.assertEqual(
'host4', utils.extract_host(weighed_host.obj.host)) 'host4', utils.extract_host(weighed_host.obj.host))
def test_capacity_weight_multiplier2(self): def test_capacity_weight_multiplier_2(self):
self.flags(capacity_weight_multiplier=2.0) self.flags(capacity_weight_multiplier=2.0)
hostinfo_list = self._get_all_hosts() hostinfo_list = self._get_all_hosts()
# host1: free_capacity_gb=1024, free=1024*(1-0.1)*2 # host1: thin_provisioning_support = False
# host2: free_capacity_gb=300, free=300*(1-0.1)*2 # free_capacity_gb = 1024
# host3: free_capacity_gb=512, free=512*2 # free = math.floor(1024-1024*0.1) = 921.0
# host4: free_capacity_gb=200, free=200*(1-0.05)*2 # free * 2 = 1842.0
# weight = 0.81
# host2: thin_provisioning_support = True
# max_over_subscription_ratio = 2.0
# free_capacity_gb = 300
# free = math.floor(2048 * 2.0 - 1748 - 2048 * 0.1) = 2143.0
# free * 2 = 4286.0
# weight = 2.0
# host3: thin_provisioning_support = False
# free_capacity_gb = 512
# free = math.floor(256 - 512 * 0) = 256.0
# free * 2 = 512.0
# weight = 0.16
# host4: thin_provisioning_support = True
# max_over_subscription_ratio = 1.0
# free_capacity_gb = 200
# free = math.floor(2048 * 1.0 - 1848 - 2048 * 0.05) = 97.0
# free * 2 = 194.0
# weight = 0.0
# host5: thin_provisioning_support = True
# max_over_subscription_ratio = 1.5
# free_capacity_gb = 500
# free = math.floor(2048 * 1.5 - 1548 - 2048 * 0.05) = 1421.0
# free * 2 = 2842.0
# weight = 1.29
# so, host1 should win: # so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list) weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 2.0) self.assertEqual(2.0, weighed_host.weight)
self.assertEqual( self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host)) 'host2', utils.extract_host(weighed_host.obj.host))

194
manila/tests/scheduler/test_host_filters.py
View File

@ -15,6 +15,7 @@
Tests For Scheduler Host Filters. Tests For Scheduler Host Filters.
""" """
import ddt
from oslo_serialization import jsonutils from oslo_serialization import jsonutils
from manila import context from manila import context
@ -24,6 +25,7 @@ from manila.tests.scheduler import fakes
from manila import utils from manila import utils
@ddt.ddt
class HostFiltersTestCase(test.TestCase): class HostFiltersTestCase(test.TestCase):
"""Test case for host filters.""" """Test case for host filters."""
@ -46,62 +48,196 @@ class HostFiltersTestCase(test.TestCase):
return ret_value return ret_value
self.mock_object(utils, 'service_is_up', fake_service_is_up) self.mock_object(utils, 'service_is_up', fake_service_is_up)
def test_capacity_filter_passes(self): @ddt.data(
{'size': 100, 'share_on': None, 'host': 'host1'},
{'size': 100, 'share_on': 'host1#pool1', 'host': 'host1#pools1'})
@ddt.unpack
def test_capacity_filter_passes(self, size, share_on, host):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': size,
'share_exists_on': share_on}
service = {'disabled': False}
host = fakes.FakeHostState(host,
{'total_capacity_gb': 500,
'free_capacity_gb': 200,
'updated_at': None,
'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
@ddt.data(
{'free_capacity': 120, 'total_capacity': 200,
'reserved': 20},
{'free_capacity': None, 'total_capacity': None,
'reserved': None})
@ddt.unpack
def test_capacity_filter_fails(self, free_capacity, total_capacity,
reserved):
self._stub_service_is_up(True) self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']() filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100} filter_properties = {'size': 100}
service = {'disabled': False} service = {'disabled': False}
host = fakes.FakeHostState('host1', host = fakes.FakeHostState('host1',
{'free_capacity_gb': 200, {'total_capacity_gb': total_capacity,
'updated_at': None, 'free_capacity_gb': free_capacity,
'service': service}) 'reserved_percentage': reserved,
self.assertTrue(filt_cls.host_passes(host, filter_properties))
def test_capacity_filter_current_host_passes(self):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100, 'share_exists_on': 'host1#pool1'}
service = {'disabled': False}
host = fakes.FakeHostState('host1#pools1',
{'free_capacity_gb': 200,
'updated_at': None,
'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
def test_capacity_filter_fails(self):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100}
service = {'disabled': False}
host = fakes.FakeHostState('host1',
{'free_capacity_gb': 120,
'reserved_percentage': 20,
'updated_at': None, 'updated_at': None,
'service': service}) 'service': service})
self.assertFalse(filt_cls.host_passes(host, filter_properties)) self.assertFalse(filt_cls.host_passes(host, filter_properties))
def test_capacity_filter_passes_infinite(self): @ddt.data('infinite', 'unknown')
def test_capacity_filter_passes_infinite_unknown(self, free):
self._stub_service_is_up(True) self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']() filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100} filter_properties = {'size': 100}
service = {'disabled': False} service = {'disabled': False}
host = fakes.FakeHostState('host1', host = fakes.FakeHostState('host1',
{'free_capacity_gb': 'infinite', {'free_capacity_gb': free,
'updated_at': None, 'updated_at': None,
'service': service}) 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties)) self.assertTrue(filt_cls.host_passes(host, filter_properties))
def test_capacity_filter_passes_unknown(self): @ddt.data(
{'free_capacity': 'infinite', 'total_capacity': 'infinite'},
{'free_capacity': 'unknown', 'total_capacity': 'unknown'})
@ddt.unpack
def test_capacity_filter_passes_total(self, free_capacity,
total_capacity):
self._stub_service_is_up(True) self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']() filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100} filter_properties = {'size': 100}
service = {'disabled': False} service = {'disabled': False}
host = fakes.FakeHostState('host1', host = fakes.FakeHostState('host1',
{'free_capacity_gb': 'unknown', {'free_capacity_gb': free_capacity,
'total_capacity_gb': total_capacity,
'reserved_percentage': 0,
'updated_at': None, 'updated_at': None,
'service': service}) 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties)) self.assertTrue(filt_cls.host_passes(host, filter_properties))
@ddt.data('infinite', 'unknown', 0)
def test_capacity_filter_fails_total(self, total):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': 100}
service = {'disabled': False}
host = fakes.FakeHostState('host1',
{'total_capacity_gb': total,
'reserved_percentage': 5,
'updated_at': None,
'service': service})
self.assertFalse(filt_cls.host_passes(host, filter_properties))
@ddt.data(
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 200, 'provisioned': 500,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 3000, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 200, 'provisioned': 7000,
'max_ratio': 20, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> False', 'cap_thick': '<is> True',
'total': 500, 'free': 200, 'provisioned': 300,
'max_ratio': 1.0, 'reserved': 5, 'thin_prov': False,
'thick_prov': True},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 200, 'provisioned': 400,
'max_ratio': 1.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> True',
'total': 500, 'free': 125, 'provisioned': 400,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': True},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 80, 'provisioned': 600,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> True',
'total': 500, 'free': 100, 'provisioned': 400,
'max_ratio': 2.0, 'reserved': 0, 'thin_prov': True,
'thick_prov': True})
@ddt.unpack
def test_filter_thin_thick_passes(self, size, cap_thin, cap_thick,
total, free, provisioned, max_ratio,
reserved, thin_prov, thick_prov):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': size,
'capabilities:thin_provisioning_support':
cap_thin,
'capabilities:thick_provisioning_support':
cap_thick}
service = {'disabled': False}
host = fakes.FakeHostState('host1',
{'total_capacity_gb': total,
'free_capacity_gb': free,
'provisioned_capacity_gb': provisioned,
'max_over_subscription_ratio': max_ratio,
'reserved_percentage': reserved,
'thin_provisioning_support': thin_prov,
'thick_provisioning_support': thick_prov,
'updated_at': None,
'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
@ddt.data(
{'size': 200, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 100, 'provisioned': 400,
'max_ratio': 0.8, 'reserved': 0, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 200, 'provisioned': 700,
'max_ratio': 1.5, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 2000, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 30, 'provisioned': 9000,
'max_ratio': 20.0, 'reserved': 0, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 100, 'provisioned': 1000,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': False},
{'size': 100, 'cap_thin': '<is> False', 'cap_thick': '<is> True',
'total': 500, 'free': 100, 'provisioned': 400,
'max_ratio': 1.0, 'reserved': 5, 'thin_prov': False,
'thick_prov': True},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> True',
'total': 500, 'free': 0, 'provisioned': 800,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': True},
{'size': 100, 'cap_thin': '<is> True', 'cap_thick': '<is> True',
'total': 500, 'free': 99, 'provisioned': 1000,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': True},
{'size': 400, 'cap_thin': '<is> True', 'cap_thick': '<is> False',
'total': 500, 'free': 200, 'provisioned': 600,
'max_ratio': 2.0, 'reserved': 5, 'thin_prov': True,
'thick_prov': False})
@ddt.unpack
def test_filter_thin_thick_fails(self, size, cap_thin, cap_thick,
total, free, provisioned, max_ratio,
reserved, thin_prov, thick_prov):
self._stub_service_is_up(True)
filt_cls = self.class_map['CapacityFilter']()
filter_properties = {'size': size,
'capabilities:thin_provisioning_support':
cap_thin,
'capabilities:thick_provisioning_support':
cap_thick}
service = {'disabled': False}
host = fakes.FakeHostState('host1',
{'total_capacity_gb': total,
'free_capacity_gb': free,
'provisioned_capacity_gb': provisioned,
'max_over_subscription_ratio': max_ratio,
'reserved_percentage': reserved,
'thin_provisioning_support': thin_prov,
'thick_provisioning_support': thick_prov,
'updated_at': None,
'service': service})
self.assertFalse(filt_cls.host_passes(host, filter_properties))
def test_retry_filter_disabled(self): def test_retry_filter_disabled(self):
# Test case where retry/re-scheduling is disabled. # Test case where retry/re-scheduling is disabled.
filt_cls = self.class_map['RetryFilter']() filt_cls = self.class_map['RetryFilter']()

48
manila/tests/scheduler/test_host_manager.py
View File

@ -188,6 +188,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 512, 'total_capacity_gb': 512,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 312,
'max_over_subscription_ratio': 1.0,
'thin_provisioning_support': False,
'thick_provisioning_support': True,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -204,6 +208,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 256, 'total_capacity_gb': 256,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 400,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -220,6 +228,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 10000, 'total_capacity_gb': 10000,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 50000,
'max_over_subscription_ratio': 20.0,
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -256,6 +268,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 51, 'total_capacity_gb': 51,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 10,
'max_over_subscription_ratio': 1.0,
'thin_provisioning_support': False,
'thick_provisioning_support': True,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -273,6 +289,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 52, 'total_capacity_gb': 52,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 60,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -290,6 +310,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 53, 'total_capacity_gb': 53,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 100,
'max_over_subscription_ratio': 20.0,
'thin_provisioning_support': True,
'thick_provisioning_support': True,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -307,6 +331,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 541, 'total_capacity_gb': 541,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 800,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -324,6 +352,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 542, 'total_capacity_gb': 542,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 2000,
'max_over_subscription_ratio': 10.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None, 'storage_protocol': None,
'driver_handles_share_servers': False, 'driver_handles_share_servers': False,
@ -374,7 +406,11 @@ class HostManagerTestCase(test.TestCase):
'total_capacity_gb': 512, 'total_capacity_gb': 512,
'reserved_percentage': 0, 'reserved_percentage': 0,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None 'storage_protocol': None,
'provisioned_capacity_gb': 312,
'max_over_subscription_ratio': 1.0,
'thin_provisioning_support': False,
'thick_provisioning_support': True,
}, },
}, { }, {
'name': 'host2@back1#BBB', 'name': 'host2@back1#BBB',
@ -390,7 +426,11 @@ class HostManagerTestCase(test.TestCase):
'total_capacity_gb': 256, 'total_capacity_gb': 256,
'reserved_percentage': 0, 'reserved_percentage': 0,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None 'storage_protocol': None,
'provisioned_capacity_gb': 400,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
}, },
}, },
] ]
@ -428,6 +468,10 @@ class HostManagerTestCase(test.TestCase):
'driver_version': None, 'driver_version': None,
'total_capacity_gb': 52, 'total_capacity_gb': 52,
'reserved_percentage': 0, 'reserved_percentage': 0,
'provisioned_capacity_gb': 60,
'max_over_subscription_ratio': 2.0,
'thin_provisioning_support': True,
'thick_provisioning_support': False,
'vendor_name': None, 'vendor_name': None,
'storage_protocol': None 'storage_protocol': None
}, },

13
manila/tests/share/test_driver.py
View File

@ -206,3 +206,16 @@ class ShareDriverTestCase(test.TestCase):
share_driver = driver.ShareDriver(value) share_driver = driver.ShareDriver(value)
self.assertEqual([], self.assertEqual([],
share_driver.get_share_server_pools('fake_server')) share_driver.get_share_server_pools('fake_server'))
@ddt.data(0.8, 1.0, 10.5, 20.0, None)
def test_check_for_setup_error(self, value):
driver.CONF.set_default('driver_handles_share_servers', False)
share_driver = driver.ShareDriver(False)
share_driver.configuration = configuration.Configuration(None)
self.mock_object(share_driver.configuration, 'safe_get',
mock.Mock(return_value=value))
if value >= 1.0:
share_driver.check_for_setup_error()
else:
self.assertRaises(exception.InvalidParameterValue,
share_driver.check_for_setup_error)