Merge "Fix and unify capacity calculations" into stable/xena
This commit is contained in:
commit
73d1473f66
|
@ -17,11 +17,10 @@
|
|||
# under the License.
|
||||
|
||||
|
||||
import math
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from cinder.scheduler import filters
|
||||
from cinder import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
@ -103,10 +102,6 @@ class CapacityFilter(filters.BaseBackendFilter):
|
|||
"grouping_name": backend_state.backend_id})
|
||||
return False
|
||||
|
||||
# Calculate how much free space is left after taking into account
|
||||
# the reserved space.
|
||||
free = free_space - math.floor(total * reserved)
|
||||
|
||||
# NOTE(xyang): If 'provisioning:type' is 'thick' in extra_specs,
|
||||
# we will not use max_over_subscription_ratio and
|
||||
# provisioned_capacity_gb to determine whether a volume can be
|
||||
|
@ -118,18 +113,45 @@ class CapacityFilter(filters.BaseBackendFilter):
|
|||
if provision_type == 'thick':
|
||||
thin = False
|
||||
|
||||
thin_support = backend_state.thin_provisioning_support
|
||||
if thin_support:
|
||||
max_over_subscription_ratio = (
|
||||
backend_state.max_over_subscription_ratio
|
||||
)
|
||||
else:
|
||||
max_over_subscription_ratio = 1
|
||||
|
||||
# NOTE(hemna): this takes into consideration all major factors
|
||||
# including reserved space, free_space (reported by driver),
|
||||
# and over subscription ratio.
|
||||
factors = utils.calculate_capacity_factors(
|
||||
total_space,
|
||||
free_space,
|
||||
backend_state.provisioned_capacity_gb,
|
||||
thin_support,
|
||||
max_over_subscription_ratio,
|
||||
backend_state.reserved_percentage,
|
||||
thin
|
||||
)
|
||||
virtual_free_space = factors["virtual_free_capacity"]
|
||||
LOG.debug("Storage Capacity factors %s", factors)
|
||||
|
||||
msg_args = {"grouping_name": backend_state.backend_id,
|
||||
"grouping": grouping,
|
||||
"requested": requested_size,
|
||||
"available": free}
|
||||
"available": virtual_free_space}
|
||||
|
||||
# Only evaluate using max_over_subscription_ratio if
|
||||
# thin_provisioning_support is True. Check if the ratio of
|
||||
# provisioned capacity over total capacity has exceeded over
|
||||
# subscription ratio.
|
||||
if (thin and backend_state.thin_provisioning_support and
|
||||
backend_state.max_over_subscription_ratio >= 1):
|
||||
provisioned_ratio = ((backend_state.provisioned_capacity_gb +
|
||||
requested_size) / total)
|
||||
provisioned_ratio = (
|
||||
(backend_state.provisioned_capacity_gb + requested_size) / (
|
||||
factors["total_available_capacity"]
|
||||
)
|
||||
)
|
||||
LOG.debug("Checking provisioning for request of %s GB. "
|
||||
"Backend: %s", requested_size, backend_state)
|
||||
if provisioned_ratio > backend_state.max_over_subscription_ratio:
|
||||
|
@ -149,14 +171,12 @@ class CapacityFilter(filters.BaseBackendFilter):
|
|||
else:
|
||||
# Thin provisioning is enabled and projected over-subscription
|
||||
# ratio does not exceed max_over_subscription_ratio. The host
|
||||
# passes if "adjusted" free virtual capacity is enough to
|
||||
# passes if virtual free capacity is enough to
|
||||
# accommodate the volume. Adjusted free virtual capacity is
|
||||
# the currently available free capacity (taking into account
|
||||
# of reserved space) which we can over-subscribe.
|
||||
adjusted_free_virtual = (
|
||||
free * backend_state.max_over_subscription_ratio)
|
||||
msg_args["available"] = adjusted_free_virtual
|
||||
res = adjusted_free_virtual >= requested_size
|
||||
msg_args["available"] = virtual_free_space
|
||||
res = virtual_free_space >= requested_size
|
||||
if not res:
|
||||
LOG.warning("Insufficient free virtual space "
|
||||
"(%(available)sGB) to accommodate thin "
|
||||
|
@ -179,7 +199,7 @@ class CapacityFilter(filters.BaseBackendFilter):
|
|||
"grouping_name": backend_state.backend_id})
|
||||
return False
|
||||
|
||||
if free < requested_size:
|
||||
if virtual_free_space < requested_size:
|
||||
LOG.warning("Insufficient free space for volume creation "
|
||||
"on %(grouping)s %(grouping_name)s (requested / "
|
||||
"avail): %(requested)s/%(available)s",
|
||||
|
|
|
@ -114,7 +114,7 @@ class CapacityWeigherTestCase(test.TestCase):
|
|||
{'volume_type': {'extra_specs': {'provisioning:type': 'thin'}},
|
||||
'winner': 'host4'},
|
||||
{'volume_type': {'extra_specs': {'provisioning:type': 'thick'}},
|
||||
'winner': 'host2'},
|
||||
'winner': 'host4'},
|
||||
{'volume_type': {'extra_specs': {}},
|
||||
'winner': 'host4'},
|
||||
{'volume_type': {},
|
||||
|
|
|
@ -99,7 +99,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
|
|||
def test_filter_fails(self, _mock_serv_is_up):
|
||||
_mock_serv_is_up.return_value = True
|
||||
filt_cls = self.class_map['CapacityFilter']()
|
||||
filter_properties = {'size': 100,
|
||||
filter_properties = {'size': 121,
|
||||
'request_spec': {'volume_id': fake.VOLUME_ID}}
|
||||
service = {'disabled': False}
|
||||
host = fakes.FakeBackendState('host1',
|
||||
|
@ -282,7 +282,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
|
|||
def test_filter_thin_true_passes2(self, _mock_serv_is_up):
|
||||
_mock_serv_is_up.return_value = True
|
||||
filt_cls = self.class_map['CapacityFilter']()
|
||||
filter_properties = {'size': 3000,
|
||||
filter_properties = {'size': 2400,
|
||||
'capabilities:thin_provisioning_support':
|
||||
'<is> True',
|
||||
'capabilities:thick_provisioning_support':
|
||||
|
@ -462,7 +462,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
|
|||
def test_filter_reserved_thin_thick_true_fails(self, _mock_serv_is_up):
|
||||
_mock_serv_is_up.return_value = True
|
||||
filt_cls = self.class_map['CapacityFilter']()
|
||||
filter_properties = {'size': 100,
|
||||
filter_properties = {'size': 151,
|
||||
'capabilities:thin_provisioning_support':
|
||||
'<is> True',
|
||||
'capabilities:thick_provisioning_support':
|
||||
|
|
|
@ -1029,7 +1029,7 @@ class HostManagerTestCase(test.TestCase):
|
|||
"free": 18.01,
|
||||
"allocated": 2.0,
|
||||
"provisioned": 2.0,
|
||||
"virtual_free": 37.02,
|
||||
"virtual_free": 36.02,
|
||||
"reported_at": 40000},
|
||||
{"name_to_id": 'host1@backend1',
|
||||
"type": "backend",
|
||||
|
@ -1037,7 +1037,7 @@ class HostManagerTestCase(test.TestCase):
|
|||
"free": 46.02,
|
||||
"allocated": 4.0,
|
||||
"provisioned": 4.0,
|
||||
"virtual_free": 64.03,
|
||||
"virtual_free": 63.03,
|
||||
"reported_at": 40000}]
|
||||
|
||||
expected2 = [
|
||||
|
@ -1055,7 +1055,7 @@ class HostManagerTestCase(test.TestCase):
|
|||
"free": 46.02,
|
||||
"allocated": 4.0,
|
||||
"provisioned": 4.0,
|
||||
"virtual_free": 95.04,
|
||||
"virtual_free": 94.04,
|
||||
"reported_at": 40000}]
|
||||
|
||||
def sort_func(data):
|
||||
|
|
|
@ -1083,10 +1083,10 @@ class TestCalculateVirtualFree(test.TestCase):
|
|||
'is_thin_lun': False, 'expected': 27.01},
|
||||
{'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': False,
|
||||
'is_thin_lun': True, 'expected': 37.02},
|
||||
'is_thin_lun': True, 'expected': 36.02},
|
||||
{'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': True,
|
||||
'is_thin_lun': True, 'expected': 37.02},
|
||||
'is_thin_lun': True, 'expected': 36.02},
|
||||
{'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': True,
|
||||
'is_thin_lun': False, 'expected': 27.01},
|
||||
|
@ -1114,6 +1114,98 @@ class TestCalculateVirtualFree(test.TestCase):
|
|||
|
||||
self.assertEqual(expected, free_capacity)
|
||||
|
||||
@ddt.data(
|
||||
{'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 1.0,
|
||||
'thin_support': False, 'thick_support': True,
|
||||
'is_thin_lun': False, 'reserved_percentage': 5,
|
||||
'expected_total_capacity': 30.01,
|
||||
'expected_reserved_capacity': 1,
|
||||
'expected_free_capacity': 28.01,
|
||||
'expected_total_available_capacity': 29.01,
|
||||
'expected_virtual_free': 27.01,
|
||||
'expected_free_percent': 93.11,
|
||||
'expected_provisioned_type': 'thick',
|
||||
'expected_provisioned_ratio': 0.07},
|
||||
{'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': False,
|
||||
'is_thin_lun': True, 'reserved_percentage': 10,
|
||||
'expected_total_capacity': 20.01,
|
||||
'expected_reserved_capacity': 2,
|
||||
'expected_free_capacity': 18.01,
|
||||
'expected_total_available_capacity': 36.02,
|
||||
'expected_virtual_free': 34.02,
|
||||
'expected_free_percent': 94.45,
|
||||
'expected_provisioned_type': 'thin',
|
||||
'expected_provisioned_ratio': 0.06},
|
||||
{'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': True,
|
||||
'is_thin_lun': True, 'reserved_percentage': 20,
|
||||
'expected_total_capacity': 20.01,
|
||||
'expected_reserved_capacity': 4,
|
||||
'expected_free_capacity': 18.01,
|
||||
'expected_total_available_capacity': 32.02,
|
||||
'expected_virtual_free': 30.02,
|
||||
'expected_free_percent': 93.75,
|
||||
'expected_provisioned_type': 'thin',
|
||||
'expected_provisioned_ratio': 0.06},
|
||||
{'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
|
||||
'thin_support': True, 'thick_support': True,
|
||||
'is_thin_lun': False, 'reserved_percentage': 10,
|
||||
'expected_total_capacity': 30.01,
|
||||
'expected_reserved_capacity': 3,
|
||||
'expected_free_capacity': 28.01,
|
||||
'expected_total_available_capacity': 27.01,
|
||||
'expected_virtual_free': 25.01,
|
||||
'expected_free_percent': 92.6,
|
||||
'expected_provisioned_type': 'thick',
|
||||
'expected_provisioned_ratio': 0.07},
|
||||
)
|
||||
@ddt.unpack
|
||||
def test_utils_calculate_capacity_factors(
|
||||
self, total, free, provisioned, max_ratio, thin_support,
|
||||
thick_support, is_thin_lun, reserved_percentage,
|
||||
expected_total_capacity,
|
||||
expected_reserved_capacity,
|
||||
expected_free_capacity,
|
||||
expected_total_available_capacity,
|
||||
expected_virtual_free,
|
||||
expected_free_percent,
|
||||
expected_provisioned_type,
|
||||
expected_provisioned_ratio):
|
||||
host_stat = {'total_capacity_gb': total,
|
||||
'free_capacity_gb': free,
|
||||
'provisioned_capacity_gb': provisioned,
|
||||
'max_over_subscription_ratio': max_ratio,
|
||||
'thin_provisioning_support': thin_support,
|
||||
'thick_provisioning_support': thick_support,
|
||||
'reserved_percentage': reserved_percentage}
|
||||
|
||||
factors = utils.calculate_capacity_factors(
|
||||
host_stat['total_capacity_gb'],
|
||||
host_stat['free_capacity_gb'],
|
||||
host_stat['provisioned_capacity_gb'],
|
||||
host_stat['thin_provisioning_support'],
|
||||
host_stat['max_over_subscription_ratio'],
|
||||
host_stat['reserved_percentage'],
|
||||
is_thin_lun)
|
||||
|
||||
self.assertEqual(expected_total_capacity,
|
||||
factors['total_capacity'])
|
||||
self.assertEqual(expected_reserved_capacity,
|
||||
factors['reserved_capacity'])
|
||||
self.assertEqual(expected_free_capacity,
|
||||
factors['free_capacity'])
|
||||
self.assertEqual(expected_total_available_capacity,
|
||||
factors['total_available_capacity'])
|
||||
self.assertEqual(expected_virtual_free,
|
||||
factors['virtual_free_capacity'])
|
||||
self.assertEqual(expected_free_percent,
|
||||
factors['free_percent'])
|
||||
self.assertEqual(expected_provisioned_type,
|
||||
factors['provisioned_type'])
|
||||
self.assertEqual(expected_provisioned_ratio,
|
||||
factors['provisioned_ratio'])
|
||||
|
||||
|
||||
class Comparable(utils.ComparableMixin):
|
||||
def __init__(self, value):
|
||||
|
|
130
cinder/utils.py
130
cinder/utils.py
|
@ -706,14 +706,118 @@ def build_or_str(elements: Union[None, str, Iterable[str]],
|
|||
return elements
|
||||
|
||||
|
||||
def calculate_capacity_factors(total_capacity: float,
|
||||
free_capacity: float,
|
||||
provisioned_capacity: float,
|
||||
thin_provisioning_support: bool,
|
||||
max_over_subscription_ratio: float,
|
||||
reserved_percentage: int,
|
||||
thin: bool) -> dict:
|
||||
"""Create the various capacity factors of the a particular backend.
|
||||
|
||||
Based off of definition of terms
|
||||
cinder-specs/specs/queens/provisioning-improvements.html
|
||||
Description of factors calculated where units of gb are Gibibytes.
|
||||
reserved_capacity - The amount of space reserved from the total_capacity
|
||||
as reported by the backend.
|
||||
total_reserved_available_capacity - The total capacity minus reserved
|
||||
capacity
|
||||
total_available_capacity - The total capacity available to cinder
|
||||
calculated from total_reserved_available_capacity (for thick) OR
|
||||
for thin total_reserved_available_capacity max_over_subscription_ratio
|
||||
calculated_free_capacity - total_available_capacity - provisioned_capacity
|
||||
virtual_free_capacity - The calculated free capacity available to cinder
|
||||
to allocate new storage.
|
||||
For thin: calculated_free_capacity
|
||||
For thick: the reported free_capacity can be less than the calculated
|
||||
capacity, so we use free_capacity - reserved_capacity.
|
||||
|
||||
free_percent - the percentage of the virtual_free_capacity and
|
||||
total_available_capacity is left over
|
||||
provisioned_ratio - The ratio of provisioned storage to
|
||||
total_available_capacity
|
||||
|
||||
:param total_capacity: The reported total capacity in the backend.
|
||||
:type total_capacity: float
|
||||
:param free_capacity: The free space/capacity as reported by the backend.
|
||||
:type free_capacity: float
|
||||
:param provisioned_capacity: as reported by backend or volume manager from
|
||||
allocated_capacity_gb
|
||||
:type provisioned_capacity: float
|
||||
:param thin_provisioning_support: Is thin provisioning supported?
|
||||
:type thin_provisioning_support: bool
|
||||
:param max_over_subscription_ratio: as reported by the backend
|
||||
:type max_over_subscription_ratio: float
|
||||
:param reserved_percentage: the % amount to reserve as unavailable. 0-100
|
||||
:type reserved_percentage: int, 0-100
|
||||
:param thin: calculate based on thin provisioning if enabled by
|
||||
thin_provisioning_support
|
||||
:type thin: bool
|
||||
:return: A dictionary of all of the capacity factors.
|
||||
:rtype: dict
|
||||
|
||||
"""
|
||||
|
||||
total = float(total_capacity)
|
||||
reserved = float(reserved_percentage) / 100
|
||||
reserved_capacity = math.floor(total * reserved)
|
||||
total_reserved_available = total - reserved_capacity
|
||||
|
||||
if thin and thin_provisioning_support:
|
||||
total_available_capacity = (
|
||||
total_reserved_available * max_over_subscription_ratio
|
||||
)
|
||||
calculated_free = total_available_capacity - provisioned_capacity
|
||||
virtual_free = calculated_free
|
||||
provisioned_type = 'thin'
|
||||
else:
|
||||
# Calculate how much free space is left after taking into
|
||||
# account the reserved space.
|
||||
total_available_capacity = total_reserved_available
|
||||
calculated_free = total_available_capacity - provisioned_capacity
|
||||
virtual_free = calculated_free
|
||||
if free_capacity < calculated_free:
|
||||
virtual_free = free_capacity
|
||||
|
||||
provisioned_type = 'thick'
|
||||
|
||||
if total_available_capacity:
|
||||
provisioned_ratio = provisioned_capacity / total_available_capacity
|
||||
free_percent = (virtual_free / total_available_capacity) * 100
|
||||
else:
|
||||
provisioned_ratio = 0
|
||||
free_percent = 0
|
||||
|
||||
def _limit(x):
|
||||
"""Limit our floating points to 2 decimal places."""
|
||||
return round(x, 2)
|
||||
|
||||
return {
|
||||
"total_capacity": total,
|
||||
"free_capacity": free_capacity,
|
||||
"reserved_capacity": reserved_capacity,
|
||||
"total_reserved_available_capacity": _limit(total_reserved_available),
|
||||
"max_over_subscription_ratio": (
|
||||
max_over_subscription_ratio if provisioned_type == 'thin' else None
|
||||
),
|
||||
"total_available_capacity": _limit(total_available_capacity),
|
||||
"provisioned_capacity": provisioned_capacity,
|
||||
"calculated_free_capacity": _limit(calculated_free),
|
||||
"virtual_free_capacity": _limit(virtual_free),
|
||||
"free_percent": _limit(free_percent),
|
||||
"provisioned_ratio": _limit(provisioned_ratio),
|
||||
"provisioned_type": provisioned_type
|
||||
}
|
||||
|
||||
|
||||
def calculate_virtual_free_capacity(total_capacity: float,
|
||||
free_capacity: float,
|
||||
provisioned_capacity: float,
|
||||
thin_provisioning_support: bool,
|
||||
max_over_subscription_ratio: float,
|
||||
reserved_percentage: float,
|
||||
reserved_percentage: int,
|
||||
thin: bool) -> float:
|
||||
"""Calculate the virtual free capacity based on thin provisioning support.
|
||||
"""Calculate the virtual free capacity based on multiple factors.
|
||||
|
||||
:param total_capacity: total_capacity_gb of a host_state or pool.
|
||||
:param free_capacity: free_capacity_gb of a host_state or pool.
|
||||
|
@ -729,18 +833,16 @@ def calculate_virtual_free_capacity(total_capacity: float,
|
|||
:returns: the calculated virtual free capacity.
|
||||
"""
|
||||
|
||||
total = float(total_capacity)
|
||||
reserved = float(reserved_percentage) / 100
|
||||
|
||||
if thin and thin_provisioning_support:
|
||||
free = (total * max_over_subscription_ratio
|
||||
- provisioned_capacity
|
||||
- math.floor(total * reserved))
|
||||
else:
|
||||
# Calculate how much free space is left after taking into
|
||||
# account the reserved space.
|
||||
free = free_capacity - math.floor(total * reserved)
|
||||
return free
|
||||
factors = calculate_capacity_factors(
|
||||
total_capacity,
|
||||
free_capacity,
|
||||
provisioned_capacity,
|
||||
thin_provisioning_support,
|
||||
max_over_subscription_ratio,
|
||||
reserved_percentage,
|
||||
thin
|
||||
)
|
||||
return factors["virtual_free_capacity"]
|
||||
|
||||
|
||||
def calculate_max_over_subscription_ratio(
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
other:
|
||||
- |
|
||||
Unified how cinder calculates the virtual free storage space for a pool.
|
||||
Previously Cinder had 2 different mechanisms for calculating the
|
||||
virtual free storage. Now both the Capacity Filter and the Capacity
|
||||
Weigher use the same mechanism, which is based upon the defined terms in
|
||||
https://specs.openstack.org/openstack/cinder-specs/specs/queens/provisioning-improvements.html
|
Loading…
Reference in New Issue