Generalize dependent object backporting

This adds some infrastructure to the base NovaObject to handle backporting
subobjects based on version rules. Here, we introduce a list of version
mapping for each sub-object and logic in the base obj_make_compatible()
to honor those.

Note that this includes a set of rules for security_group_rule because
there is a test that ends up with a call to obj_make_compatible() that
complains if it's not there.

Related to blueprint kilo-objects

Change-Id: I9582a8519ceb7e131550aa97ef16995da6dd1ff2

nova/objects/base.py

@@ -32,6 +32,7 @@ from nova import objects
 from nova.objects import fields
 from nova.openstack.common import log as logging
 from nova.openstack.common import versionutils
+from nova import utils
 
 
 LOG = logging.getLogger('object')
@@ -235,6 +236,32 @@ class NovaObject(object):
     fields = {}
     obj_extra_fields = []
 
+    # Table of sub-object versioning information
+    #
+    # This contains a list of version mappings, by the field name of
+    # the subobject. The mappings must be in order of oldest to
+    # newest, and are tuples of (my_version, subobject_version). A
+    # request to backport this object to $my_version will cause the
+    # subobject to be backported to $subobject_version.
+    #
+    # obj_relationships = {
+    #     'subobject1': [('1.2', '1.1'), ('1.4', '1.2')],
+    #     'subobject2': [('1.2', '1.0')],
+    # }
+    #
+    # In the above example:
+    #
+    # - If we are asked to backport our object to version 1.3,
+    #   subobject1 will be backported to version 1.1, since it was
+    #   bumped to version 1.2 when our version was 1.4.
+    # - If we are asked to backport our object to version 1.5,
+    #   no changes will be made to subobject1 or subobject2, since
+    #   they have not changed since version 1.4.
+    # - If we are asked to backlevel our object to version 1.1, we
+    #   will remove both subobject1 and subobject2 from the primitive,
+    #   since they were not added until version 1.2.
+    obj_relationships = {}
+
     def __init__(self, context=None, **kwargs):
         self._changed_fields = set()
         self._context = context
@@ -337,6 +364,51 @@ class NovaObject(object):
         """Create a copy."""
         return copy.deepcopy(self)
 
+    def _obj_make_obj_compatible(self, primitive, target_version, field):
+        """Backlevel a sub-object based on our versioning rules.
+
+        This is responsible for backporting objects contained within
+        this object's primitive according to a set of rules we
+        maintain about version dependencies between objects. This
+        requires that the obj_relationships table in this object is
+        correct and up-to-date.
+
+        :param:primitive: The primitive version of this object
+        :param:target_version: The version string requested for this object
+        :param:field: The name of the field in this object containing the
+                      sub-object to be backported
+        """
+
+        def _do_backport(to_version):
+            obj = getattr(self, field)
+            if obj:
+                obj.obj_make_compatible(
+                    primitive[field]['nova_object.data'],
+                    to_version)
+                primitive[field]['nova_object.version'] = to_version
+
+        target_version = utils.convert_version_to_tuple(target_version)
+        for index, versions in enumerate(self.obj_relationships[field]):
+            my_version, child_version = versions
+            my_version = utils.convert_version_to_tuple(my_version)
+            if target_version < my_version:
+                if index == 0:
+                    # We're backporting to a version from before this
+                    # subobject was added: delete it from the primitive.
+                    del primitive[field]
+                else:
+                    # We're in the gap between index-1 and index, so
+                    # backport to the older version
+                    last_child_version = \
+                        self.obj_relationships[field][index - 1][1]
+                    _do_backport(last_child_version)
+                return
+            elif target_version == my_version:
+                # This is the first mapping that satisfies the
+                # target_version request: backport the object.
+                _do_backport(child_version)
+                return
+
     def obj_make_compatible(self, primitive, target_version):
         """Make an object representation compatible with a target version.
 
@@ -363,7 +435,18 @@ class NovaObject(object):
         :raises: nova.exception.UnsupportedObjectError if conversion
         is not possible for some reason
         """
-        pass
+        for key, field in self.fields.items():
+            if not isinstance(field, fields.ObjectField):
+                continue
+            if not self.obj_attr_is_set(key):
+                continue
+            if key not in self.obj_relationships:
+                # NOTE(danms): This is really a coding error and shouldn't
+                # happen unless we miss something
+                raise exception.ObjectActionError(
+                    action='obj_make_compatible',
+                    reason='No rule for %s' % key)
+            self._obj_make_obj_compatible(primitive, target_version, key)
 
     def obj_to_primitive(self, target_version=None):
         """Simple base-case dehydration.

nova/objects/security_group_rule.py

@@ -36,6 +36,11 @@ class SecurityGroupRule(base.NovaPersistentObject, base.NovaObject):
         'grantee_group': fields.ObjectField('SecurityGroup', nullable=True),
         }
 
+    obj_relationships = {
+        'parent_group': [('1.0', '1.1'), ('1.1', '1.1')],
+        'grantee_group': [('1.0', '1.1'), ('1.1', '1.1')],
+    }
+
     @staticmethod
     def _from_db_subgroup(context, db_group):
         if db_group is None:

nova/tests/unit/objects/test_objects.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import contextlib
+import copy
 import datetime
 import hashlib
 import inspect
@@ -102,6 +103,7 @@ class MyObj(base.NovaPersistentObject, base.NovaObject):
         self.rel_object = MyOwnedObject(baz=42)
 
     def obj_make_compatible(self, primitive, target_version):
+        super(MyObj, self).obj_make_compatible(primitive, target_version)
         # NOTE(danms): Simulate an older version that had a different
         # format for the 'bar' attribute
         if target_version == '1.1' and 'bar' in primitive:
@@ -724,6 +726,71 @@ class _TestObject(object):
                          'deleted_at=<?>,foo=123,missing=<?>,readonly=<?>,'
                          'rel_object=<?>,updated_at=<?>)', repr(obj))
 
+    def test_obj_make_obj_compatible(self):
+        subobj = MyOwnedObject(baz=1)
+        obj = MyObj(rel_object=subobj)
+        obj.obj_relationships = {
+            'rel_object': [('1.5', '1.1'), ('1.7', '1.2')],
+        }
+        primitive = obj.obj_to_primitive()['nova_object.data']
+        with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
+            obj._obj_make_obj_compatible(copy.copy(primitive), '1.8',
+                                         'rel_object')
+            self.assertFalse(mock_compat.called)
+
+        with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
+            obj._obj_make_obj_compatible(copy.copy(primitive),
+                                         '1.7', 'rel_object')
+            mock_compat.assert_called_once_with(
+                primitive['rel_object']['nova_object.data'], '1.2')
+            self.assertEqual('1.2',
+                             primitive['rel_object']['nova_object.version'])
+
+        with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
+            obj._obj_make_obj_compatible(copy.copy(primitive),
+                                         '1.6', 'rel_object')
+            mock_compat.assert_called_once_with(
+                primitive['rel_object']['nova_object.data'], '1.1')
+            self.assertEqual('1.1',
+                             primitive['rel_object']['nova_object.version'])
+
+        with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
+            obj._obj_make_obj_compatible(copy.copy(primitive), '1.5',
+                                         'rel_object')
+            mock_compat.assert_called_once_with(
+                primitive['rel_object']['nova_object.data'], '1.1')
+            self.assertEqual('1.1',
+                             primitive['rel_object']['nova_object.version'])
+
+        with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
+            _prim = copy.copy(primitive)
+            obj._obj_make_obj_compatible(_prim, '1.4', 'rel_object')
+            self.assertFalse(mock_compat.called)
+            self.assertNotIn('rel_object', _prim)
+
+    def test_obj_make_compatible_hits_sub_objects(self):
+        subobj = MyOwnedObject(baz=1)
+        obj = MyObj(foo=123, rel_object=subobj)
+        obj.obj_relationships = {'rel_object': [('1.0', '1.0')]}
+        with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat:
+            obj.obj_make_compatible({'rel_object': 'foo'}, '1.10')
+            mock_compat.assert_called_once_with({'rel_object': 'foo'}, '1.10',
+                                                'rel_object')
+
+    def test_obj_make_compatible_skips_unset_sub_objects(self):
+        obj = MyObj(foo=123)
+        obj.obj_relationships = {'rel_object': [('1.0', '1.0')]}
+        with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat:
+            obj.obj_make_compatible({'rel_object': 'foo'}, '1.10')
+            self.assertFalse(mock_compat.called)
+
+    def test_obj_make_compatible_complains_about_missing_rules(self):
+        subobj = MyOwnedObject(baz=1)
+        obj = MyObj(foo=123, rel_object=subobj)
+        obj.obj_relationships = {}
+        self.assertRaises(exception.ObjectActionError,
+                          obj.obj_make_compatible, {}, '1.0')
+
 
 class TestObject(_LocalTest, _TestObject):
     pass
@@ -1124,3 +1191,26 @@ class TestObjectVersions(test.TestCase):
                 LOG.info('testing obj: %s version: %s' %
                          (obj_name, test_version))
                 obj_class().obj_to_primitive(target_version=test_version)
+
+    def test_obj_relationships_in_order(self):
+        # Iterate all object classes and verify that we can run
+        # obj_make_compatible with every older version than current.
+        # This doesn't actually test the data conversions, but it at least
+        # makes sure the method doesn't blow up on something basic like
+        # expecting the wrong version format.
+        for obj_name in base.NovaObject._obj_classes:
+            obj_class = base.NovaObject._obj_classes[obj_name][0]
+            for field, versions in obj_class.obj_relationships.items():
+                last_my_version = (0, 0)
+                last_child_version = (0, 0)
+                for my_version, child_version in versions:
+                    _my_version = utils.convert_version_to_tuple(my_version)
+                    _ch_version = utils.convert_version_to_tuple(child_version)
+                    self.assertTrue((last_my_version < _my_version
+                                     and last_child_version <= _ch_version),
+                                    'Object %s relationship '
+                                    '%s->%s for field %s is out of order' % (
+                                        obj_name, my_version, child_version,
+                                        field))
+                    last_my_version = _my_version
+                    last_child_version = _ch_version