Merge "Add compute_node ratio online data migration script"

nova/cmd/manage.py

@@ -59,6 +59,7 @@ from nova.i18n import _
 from nova import objects
 from nova.objects import block_device as block_device_obj
 from nova.objects import build_request as build_request_obj
+from nova.objects import compute_node as compute_node_obj
 from nova.objects import host_mapping as host_mapping_obj
 from nova.objects import instance as instance_obj
 from nova.objects import instance_mapping as instance_mapping_obj
@@ -418,6 +419,8 @@ class DbCommands(object):
         consumer_obj.create_incomplete_consumers,
         # Added in Rocky
         instance_mapping_obj.populate_queued_for_delete,
+        # Added in Stein
+        compute_node_obj.migrate_empty_ratio,
     )
 
     def __init__(self):

nova/objects/compute_node.py

@@ -16,6 +16,8 @@
 from oslo_serialization import jsonutils
 from oslo_utils import uuidutils
 from oslo_utils import versionutils
+from sqlalchemy import or_
+from sqlalchemy.sql import null
 
 import nova.conf
 from nova.db import api as db
@@ -469,3 +471,39 @@ class ComputeNodeList(base.ObjectListBase, base.NovaObject):
         db_computes = cls._db_compute_node_get_by_hv_type(context, hv_type)
         return base.obj_make_list(context, cls(context), objects.ComputeNode,
                                   db_computes)
+
+
+def _get_node_empty_ratio(context, max_count):
+    """Query the DB for non-deleted compute_nodes with 0.0/None alloc ratios
+
+    Results are limited by ``max_count``.
+    """
+    return context.session.query(models.ComputeNode).filter(or_(
+        models.ComputeNode.ram_allocation_ratio == '0.0',
+        models.ComputeNode.cpu_allocation_ratio == '0.0',
+        models.ComputeNode.disk_allocation_ratio == '0.0',
+        models.ComputeNode.ram_allocation_ratio == null(),
+        models.ComputeNode.cpu_allocation_ratio == null(),
+        models.ComputeNode.disk_allocation_ratio == null()
+    )).filter(models.ComputeNode.deleted == 0).limit(max_count).all()
+
+
+@sa_api.pick_context_manager_writer
+def migrate_empty_ratio(context, max_count):
+    cns = _get_node_empty_ratio(context, max_count)
+
+    # NOTE(yikun): If it's an existing record with 0.0 or None values,
+    # we need to migrate this record using 'xxx_allocation_ratio' config
+    # if it's set, and fallback to use the 'initial_xxx_allocation_ratio'
+    # otherwise.
+    for cn in cns:
+        for t in ['cpu', 'disk', 'ram']:
+            current_ratio = getattr(cn, '%s_allocation_ratio' % t)
+            if current_ratio in (0.0, None):
+                r = getattr(CONF, "%s_allocation_ratio" % t)
+                init_x_ratio = getattr(CONF, "initial_%s_allocation_ratio" % t)
+                conf_alloc_ratio = r if r else init_x_ratio
+                setattr(cn, '%s_allocation_ratio' % t, conf_alloc_ratio)
+        context.session.add(cn)
+    found = done = len(cns)
+    return found, done

nova/tests/functional/db/test_compute_node.py

@@ -15,6 +15,7 @@ import nova.conf
 from nova import context
 from nova.db import api as db
 from nova import objects
+from nova.objects import compute_node
 from nova.objects import fields as obj_fields
 from nova import test
 
@@ -188,3 +189,71 @@ class ComputeNodeTestCase(test.TestCase):
             # the ram ratio is refreshed to CONF.initial_xxx_allocation_ratio
             self.assertEqual(CONF.initial_ram_allocation_ratio,
                              cn['ram_allocation_ratio'])
+
+    def test_migrate_empty_ratio(self):
+        # we have 5 records to process, the last of which is deleted
+        for i in range(5):
+            cn = fake_compute_obj.obj_clone()
+            cn._context = self.context
+            cn.host += '-alt-%s' % i
+            cn.create()
+            db.compute_node_update(self.context, cn.id,
+                                   {'cpu_allocation_ratio': 0.0})
+            if i == 4:
+                cn.destroy()
+
+        # first only process 2
+        res = compute_node.migrate_empty_ratio(self.context, 2)
+        self.assertEqual(res, (2, 2))
+
+        # then process others - there should only be 2 found since one
+        # of the remaining compute nodes is deleted and gets filtered out
+        res = compute_node.migrate_empty_ratio(self.context, 999)
+        self.assertEqual(res, (2, 2))
+
+    def test_migrate_none_or_zero_ratio_with_none_ratio_conf(self):
+        cn1 = fake_compute_obj.obj_clone()
+        cn1._context = self.context
+        cn1.create()
+
+        db.compute_node_update(self.context, cn1.id,
+                               {'cpu_allocation_ratio': 0.0,
+                                'disk_allocation_ratio': 0.0,
+                                'ram_allocation_ratio': 0.0})
+
+        self.flags(initial_cpu_allocation_ratio=32.0)
+        self.flags(initial_ram_allocation_ratio=8.0)
+        self.flags(initial_disk_allocation_ratio=2.0)
+
+        res = compute_node.migrate_empty_ratio(self.context, 1)
+        self.assertEqual(res, (1, 1))
+
+        # the ratio is refreshed to CONF.initial_xxx_allocation_ratio
+        # beacause CONF.xxx_allocation_ratio is None
+        cns = db.compute_node_get_all(self.context)
+        # the ratio is refreshed to CONF.xxx_allocation_ratio
+        for cn in cns:
+            for x in ['cpu', 'disk', 'ram']:
+                conf_key = 'initial_%s_allocation_ratio' % x
+                key = '%s_allocation_ratio' % x
+                self.assertEqual(getattr(CONF, conf_key), cn[key])
+
+    def test_migrate_none_or_zero_ratio_with_not_empty_ratio(self):
+        cn1 = fake_compute_obj.obj_clone()
+        cn1._context = self.context
+        cn1.create()
+
+        db.compute_node_update(self.context, cn1.id,
+                               {'cpu_allocation_ratio': 32.0,
+                                'ram_allocation_ratio': 4.0,
+                                'disk_allocation_ratio': 3.0})
+
+        res = compute_node.migrate_empty_ratio(self.context, 1)
+        # the non-empty ratio will not be refreshed
+        self.assertEqual(res, (0, 0))
+
+        cns = db.compute_node_get_all(self.context)
+        for cn in cns:
+            self.assertEqual(32.0, cn['cpu_allocation_ratio'])
+            self.assertEqual(4.0, cn['ram_allocation_ratio'])
+            self.assertEqual(3.0, cn['disk_allocation_ratio'])

releasenotes/notes/add_initial_allocation_ratio-2d2666d62426a4bf.yaml

@@ -17,6 +17,11 @@ upgrade:
     initially creating the ``computes_nodes`` table record for a given
     nova-compute service.
 
+    Existing ``compute_nodes`` table records with ``0.0`` or ``None`` values
+    for ``cpu_allocation_ratio``, ``ram_allocation_ratio`` or
+    ``disk_allocation_ratio`` will be migrated online when accessed or when
+    the ``nova-manage db online_data_migrations`` command is run.
+
     For more details, refer to the `spec`__.
 
     .. __: https://specs.openstack.org/openstack/nova-specs/specs/stein/approved/initial-allocation-ratios.html