openstack
/
nova
Code Issues Proposed changes

Add ratio online data migration when load compute node 

This patch adds an online data migrations for any compute_nodes
with existing ``0.0`` and ``None`` allocation ratio. If it's an
existing record with 0.0 values, we will replace it with the
configure ``xxx_allocation_ratio`` config if it's not None, and
fallback to using the ``initial_xxx_allocation_ratio`` otherwise.

Change-Id: Ic137d837c7b1d29f56a45a01b3b104d6d2c698df
blueprint: initial-allocation-ratios
This commit is contained in:
Yikun Jiang 2018-10-26 11:51:20 +08:00 committed by Matt Riedemann
parent 08f3ae9606
commit 4722e7116f
4 changed files with 145 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
nova/objects/compute_node.py
View File

@ -173,6 +173,7 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject):
'pci_device_pools',
])
fields = set(compute.fields) - special_cases
online_updates = {}
for key in fields:
value = db_compute[key]
# NOTE(sbauza): Since all compute nodes don't possibly run the
@ -184,36 +185,31 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject):
# the next release (Newton) where the opt default values will be
# restored for both cpu (16.0), ram (1.5) and disk (1.0)
# allocation ratios.
# TODO(sbauza): Remove that in the next major version bump where
# we break compatibility with old Liberty computes
if (key == 'cpu_allocation_ratio' or key == 'ram_allocation_ratio'
or key == 'disk_allocation_ratio'):
if value == 0.0:
# Operator has not yet provided a new value for that ratio
# on the compute node
value = None
if value is None:
# ResourceTracker is not updating the value (old node)
# or the compute node is updated but the default value has
# not been changed
value = getattr(CONF, key)
if value in (0.0, None) \
and key == 'cpu_allocation_ratio':
# It's not specified either on the controller
value = 16.0
if value in (0.0, None) \
and key == 'ram_allocation_ratio':
# It's not specified either on the controller
value = 1.5
if value in (0.0, None) \
and key == 'disk_allocation_ratio':
# It's not specified either on the controller
value = 1.0
# TODO(yikun): Remove this online migration code when all ratio
# values are NOT 0.0 or NULL
ratio_keys = ['cpu_allocation_ratio', 'ram_allocation_ratio',
'disk_allocation_ratio']
if key in ratio_keys and value in (None, 0.0):
# ResourceTracker is not updating the value (old node)
# or the compute node is updated but the default value has
# not been changed
r = getattr(CONF, key)
# NOTE(yikun): If the allocation ratio record is not set, the
# allocation ratio will be changed to the
# CONF.x_allocation_ratio value if x_allocation_ratio is
# set, and fallback to use the CONF.initial_x_allocation_ratio
# otherwise.
init_x_ratio = getattr(CONF, 'initial_%s' % key)
value = r if r else init_x_ratio
online_updates[key] = value
elif key == 'mapped':
value = 0 if value is None else value
setattr(compute, key, value)
if online_updates:
db.compute_node_update(context, compute.id, online_updates)
stats = db_compute['stats']
if stats:
compute.stats = jsonutils.loads(stats)

53
nova/tests/functional/db/test_compute_node.py
View File

@ -11,11 +11,15 @@
# under the License.
from oslo_utils.fixture import uuidsentinel
import nova.conf
from nova import context
from nova.db import api as db
from nova import objects
from nova.objects import fields as obj_fields
from nova import test
CONF = nova.conf.CONF
_HOSTNAME = 'fake-host'
_NODENAME = 'fake-node'
@ -76,6 +80,33 @@ class ComputeNodeTestCase(test.TestCase):
super(ComputeNodeTestCase, self).setUp()
self.context = context.RequestContext('fake-user', 'fake-project')
def _create_zero_and_none_cn(self):
cn1 = fake_compute_obj.obj_clone()
cn1._context = self.context
cn1.create()
db.compute_node_update(self.context, cn1.id,
{'cpu_allocation_ratio': 0.0,
'disk_allocation_ratio': 0.0,
'ram_allocation_ratio': 0.0})
cn1_db = db.compute_node_get(self.context, cn1.id)
for x in ['cpu', 'disk', 'ram']:
self.assertEqual(0.0, cn1_db['%s_allocation_ratio' % x])
cn2 = fake_compute_obj.obj_clone()
cn2._context = self.context
cn2.host += '-alt'
cn2.create()
# We can't set a cn_obj.xxx_allocation_ratio to None,
# so we set ratio to None in db directly
db.compute_node_update(self.context, cn2.id,
{'cpu_allocation_ratio': None,
'disk_allocation_ratio': None,
'ram_allocation_ratio': None})
cn2_db = db.compute_node_get(self.context, cn2.id)
for x in ['cpu', 'disk', 'ram']:
self.assertIsNone(None, cn2_db['%s_allocation_ratio' % x])
def test_get_all_by_uuids(self):
cn1 = fake_compute_obj.obj_clone()
cn1._context = self.context
@ -135,3 +166,25 @@ class ComputeNodeTestCase(test.TestCase):
'ironic')
self.assertEqual(1, len(cns))
self.assertEqual(cn1.uuid, cns[0].uuid)
def test_ratio_online_migration_when_load(self):
# set cpu and disk, and leave ram unset(None)
self.flags(cpu_allocation_ratio=1.0)
self.flags(disk_allocation_ratio=2.0)
self._create_zero_and_none_cn()
# trigger online migration
objects.ComputeNodeList.get_all(self.context)
cns = db.compute_node_get_all(self.context)
for cn in cns:
# the cpu/disk ratio is refreshed to CONF.xxx_allocation_ratio
self.assertEqual(CONF.cpu_allocation_ratio,
cn['cpu_allocation_ratio'])
self.assertEqual(CONF.disk_allocation_ratio,
cn['disk_allocation_ratio'])
# the ram ratio is refreshed to CONF.initial_xxx_allocation_ratio
self.assertEqual(CONF.initial_ram_allocation_ratio,
cn['ram_allocation_ratio'])

2
nova/tests/unit/cmd/test_status.py
View File

@ -654,7 +654,6 @@ class TestUpgradeCheckResourceProviders(test.NoDBTestCase):
# create a deleted compute node record (shouldn't count)
cn2 = objects.ComputeNode(
context=ctxt,
deleted=1,
host='fakehost',
vcpus=4,
memory_mb=8 * 1024,
@ -666,6 +665,7 @@ class TestUpgradeCheckResourceProviders(test.NoDBTestCase):
hypervisor_version=1,
cpu_info='{"arch": "x86_64"}')
cn2.create()
cn2.destroy()
# create a single resource provider with some VCPU inventory
self._create_resource_provider(FAKE_VCPU_INVENTORY)

83
nova/tests/unit/objects/test_compute_node.py
View File

@ -22,6 +22,7 @@ from oslo_utils import timeutils
from oslo_versionedobjects import base as ovo_base
from oslo_versionedobjects import exception as ovo_exc
from nova import conf
from nova.db import api as db
from nova import exception
from nova import objects
@ -131,6 +132,8 @@ fake_compute_with_resources = objects.ComputeNode(
supported_hv_specs=fake_supported_hv_specs,
)
CONF = conf.CONF
class _TestComputeNodeObject(object):
def supported_hv_specs_comparator(self, expected, obj_val):
@ -550,7 +553,13 @@ class _TestComputeNodeObject(object):
primitive = compute.obj_to_primitive(target_version='1.15')
self.assertNotIn('disk_allocation_ratio', primitive)
def test_compat_allocation_ratios_old_compute(self):
@mock.patch('nova.db.api.compute_node_update')
def test_compat_allocation_ratios_old_compute(self, mock_update):
"""Tests the scenario that allocation ratios are overridden in config
and the legacy compute node record from the database has None set for
the allocation ratio values. The result is that the migrated record
allocation ratios should reflect the config overrides.
"""
self.flags(cpu_allocation_ratio=2.0, ram_allocation_ratio=3.0,
disk_allocation_ratio=0.9)
compute_dict = fake_compute_node.copy()
@ -565,7 +574,19 @@ class _TestComputeNodeObject(object):
self.assertEqual(3.0, compute.ram_allocation_ratio)
self.assertEqual(0.9, compute.disk_allocation_ratio)
def test_compat_allocation_ratios_zero_conf(self):
mock_update.assert_called_once_with(
self.context, 123, {'cpu_allocation_ratio': 2.0,
'ram_allocation_ratio': 3.0,
'disk_allocation_ratio': 0.9})
@mock.patch('nova.db.api.compute_node_update')
def test_compat_allocation_ratios_zero_conf(self, mock_update):
"""Tests that the override allocation ratios are set to 0.0 for
whatever reason (maybe an old nova.conf sample file is being used)
and the legacy compute node record has None for allocation ratios,
so the resulting data migration makes the record allocation ratios
use the CONF.initial_*_allocation_ratio values.
"""
self.flags(cpu_allocation_ratio=0.0, ram_allocation_ratio=0.0,
disk_allocation_ratio=0.0)
compute_dict = fake_compute_node.copy()
@ -576,11 +597,25 @@ class _TestComputeNodeObject(object):
cls = objects.ComputeNode
compute = cls._from_db_object(self.context, cls(), compute_dict)
self.assertEqual(16.0, compute.cpu_allocation_ratio)
self.assertEqual(1.5, compute.ram_allocation_ratio)
self.assertEqual(1.0, compute.disk_allocation_ratio)
self.assertEqual(
CONF.initial_cpu_allocation_ratio, compute.cpu_allocation_ratio)
self.assertEqual(
CONF.initial_ram_allocation_ratio, compute.ram_allocation_ratio)
self.assertEqual(
CONF.initial_disk_allocation_ratio, compute.disk_allocation_ratio)
def test_compat_allocation_ratios_None_conf_zero_values(self):
mock_update.assert_called_once_with(
self.context, 123, {'cpu_allocation_ratio': 16.0,
'ram_allocation_ratio': 1.5,
'disk_allocation_ratio': 1.0})
@mock.patch('nova.db.api.compute_node_update')
def test_compat_allocation_ratios_None_conf_zero_values(self, mock_update):
"""Tests the scenario that the CONF.*_allocation_ratio overrides are
left to the default (None) and the compute node record allocation
ratio values in the DB are 0.0, so they will be migrated to the
CONF.initial_*_allocation_ratio values.
"""
# the CONF.x_allocation_ratio is None by default
compute_dict = fake_compute_node.copy()
# the computes provide allocation ratios 0.0
@ -590,11 +625,25 @@ class _TestComputeNodeObject(object):
cls = objects.ComputeNode
compute = cls._from_db_object(self.context, cls(), compute_dict)
self.assertEqual(16.0, compute.cpu_allocation_ratio)
self.assertEqual(1.5, compute.ram_allocation_ratio)
self.assertEqual(1.0, compute.disk_allocation_ratio)
self.assertEqual(
CONF.initial_cpu_allocation_ratio, compute.cpu_allocation_ratio)
self.assertEqual(
CONF.initial_ram_allocation_ratio, compute.ram_allocation_ratio)
self.assertEqual(
CONF.initial_disk_allocation_ratio, compute.disk_allocation_ratio)
def test_compat_allocation_ratios_None_conf_None_values(self):
mock_update.assert_called_once_with(
self.context, 123, {'cpu_allocation_ratio': 16.0,
'ram_allocation_ratio': 1.5,
'disk_allocation_ratio': 1.0})
@mock.patch('nova.db.api.compute_node_update')
def test_compat_allocation_ratios_None_conf_None_values(self, mock_update):
"""Tests the scenario that the override CONF.*_allocation_ratio options
are the default values (None), the compute node record from the DB has
None values for allocation ratios, so the resulting migrated record
will have the CONF.initial_*_allocation_ratio values.
"""
# the CONF.x_allocation_ratio is None by default
compute_dict = fake_compute_node.copy()
# # the computes provide allocation ratios None
@ -604,9 +653,17 @@ class _TestComputeNodeObject(object):
cls = objects.ComputeNode
compute = cls._from_db_object(self.context, cls(), compute_dict)
self.assertEqual(16.0, compute.cpu_allocation_ratio)
self.assertEqual(1.5, compute.ram_allocation_ratio)
self.assertEqual(1.0, compute.disk_allocation_ratio)
self.assertEqual(
CONF.initial_cpu_allocation_ratio, compute.cpu_allocation_ratio)
self.assertEqual(
CONF.initial_ram_allocation_ratio, compute.ram_allocation_ratio)
self.assertEqual(
CONF.initial_disk_allocation_ratio, compute.disk_allocation_ratio)
mock_update.assert_called_once_with(
self.context, 123, {'cpu_allocation_ratio': 16.0,
'ram_allocation_ratio': 1.5,
'disk_allocation_ratio': 1.0})
def test_get_all_by_not_mapped(self):
for mapped in (1, 0, 1, 3):