Import serialization and nesting from Nova Objects

This change mostly merges the following commits from Nova:

e91c3d141c957485dcb66c73e84b41b775e4268b
f1c4b8e5f34eb6b5e70da6711750dcf05cea8c0a
65f6c536fecd3c788b2e0dfa9d66ecd24ca550e1
92a3190128547403dc603e5a40e377c6eb0c8025
68cb4d53385821c3ffdc40c299a77d11a7f98f27

Change-Id: I0a16f45674f5d14f458e2bb490d909a9086ea8b4

ironic/objects/base.py

@@ -37,10 +37,10 @@ def make_class_properties(cls):
     cls.fields.update(IronicObject.fields)
     for name, typefn in cls.fields.iteritems():
 
-        def getter(self, name=name, typefn=typefn):
+        def getter(self, name=name):
             attrname = get_attrname(name)
             if not hasattr(self, attrname):
-                self.obj_load(name)
+                self.obj_load_attr(name)
             return getattr(self, attrname)
 
         def setter(self, value, name=name, typefn=typefn):
@@ -175,9 +175,10 @@ class IronicObject(object):
     # by subclasses, but that is a special case. Objects inheriting from
     # other objects will not receive this merging of fields contents.
     fields = {
-        'created_at': obj_utils.datetime_or_none,
+        'created_at': obj_utils.datetime_or_str_or_none,
-        'updated_at': obj_utils.datetime_or_none,
+        'updated_at': obj_utils.datetime_or_str_or_none,
         }
+    obj_extra_fields = []
 
     def __init__(self):
         self._changed_fields = set()
@@ -216,7 +217,6 @@ class IronicObject(object):
 
     _attr_created_at_from_primitive = obj_utils.dt_deserializer
     _attr_updated_at_from_primitive = obj_utils.dt_deserializer
-    _attr_deleted_at_from_primitive = obj_utils.dt_deserializer
 
     def _attr_from_primitive(self, attribute, value):
         """Attribute deserialization dispatcher.
@@ -231,7 +231,7 @@ class IronicObject(object):
         return value
 
     @classmethod
-    def obj_from_primitive(cls, primitive):
+    def obj_from_primitive(cls, primitive, context=None):
         """Simple base-case hydration.
 
         This calls self._attr_from_primitive() for each item in fields.
@@ -247,6 +247,7 @@ class IronicObject(object):
         objdata = primitive['ironic_object.data']
         objclass = cls.obj_class_from_name(objname, objver)
         self = objclass()
+        self._context = context
         for name in self.fields:
             if name in objdata:
                 setattr(self, name,
@@ -257,7 +258,6 @@ class IronicObject(object):
 
     _attr_created_at_to_primitive = obj_utils.dt_serializer('created_at')
     _attr_updated_at_to_primitive = obj_utils.dt_serializer('updated_at')
-    _attr_deleted_at_to_primitive = obj_utils.dt_serializer('deleted_at')
 
     def _attr_to_primitive(self, attribute):
         """Attribute serialization dispatcher.
@@ -307,7 +307,7 @@ class IronicObject(object):
         raise NotImplementedError('Cannot save anything in the base class')
 
     def obj_what_changed(self):
-        """Returns a list of fields that have been modified."""
+        """Returns a set of fields that have been modified."""
         return self._changed_fields
 
     def obj_reset_changes(self, fields=None):
@@ -326,8 +326,9 @@ class IronicObject(object):
 
         NOTE(danms): May be removed in the future.
         """
-        for name in self.fields:
+        for name in self.fields.keys() + self.obj_extra_fields:
-            if hasattr(self, get_attrname(name)):
+            if (hasattr(self, get_attrname(name)) or
+                    name in self.obj_extra_fields):
                 yield name, getattr(self, name)
 
     items = lambda self: list(self.iteritems())
@@ -346,6 +347,13 @@ class IronicObject(object):
         """
         setattr(self, name, value)
 
+    def __contains__(self, name):
+        """For backwards-compatibility with dict-based objects.
+
+        NOTE(danms): May be removed in the future.
+        """
+        return hasattr(self, get_attrname(name))
+
     def get(self, key, value=None):
         """For backwards-compatibility with dict-based objects.
 
@@ -353,6 +361,71 @@ class IronicObject(object):
         """
         return self[key]
 
+    def update(self, updates):
+        """For backwards-compatibility with dict-base objects.
+
+        NOTE(danms): May be removed in the future.
+        """
+        for key, value in updates.items():
+            self[key] = value
+
+
+class ObjectListBase(object):
+    """Mixin class for lists of objects.
+
+    This mixin class can be added as a base class for an object that
+    is implementing a list of objects. It adds a single field of 'objects',
+    which is the list store, and behaves like a list itself. It supports
+    serialization of the list of objects automatically.
+    """
+    fields = {
+        'objects': list,
+        }
+
+    def __iter__(self):
+        """List iterator interface."""
+        return iter(self.objects)
+
+    def __len__(self):
+        """List length."""
+        return len(self.objects)
+
+    def __getitem__(self, index):
+        """List index access."""
+        if isinstance(index, slice):
+            new_obj = self.__class__()
+            new_obj.objects = self.objects[index]
+            # NOTE(danms): We must be mixed in with an IronicObject!
+            new_obj.obj_reset_changes()
+            new_obj._context = self._context
+            return new_obj
+        return self.objects[index]
+
+    def __contains__(self, value):
+        """List membership test."""
+        return value in self.objects
+
+    def count(self, value):
+        """List count of value occurrences."""
+        return self.objects.count(value)
+
+    def index(self, value):
+        """List index of value."""
+        return self.objects.index(value)
+
+    def _attr_objects_to_primitive(self):
+        """Serialization of object list."""
+        return [x.obj_to_primitive() for x in self.objects]
+
+    def _attr_objects_from_primitive(self, value):
+        """Deserialization of object list."""
+        objects = []
+        for entity in value:
+            obj = IronicObject.obj_from_primitive(entity,
+                                                  context=self._context)
+            objects.append(obj)
+        return objects
+
 
 class IronicObjectSerializer(rpc_serializer.Serializer):
     """A IronicObject-aware Serializer.
@@ -362,14 +435,53 @@ class IronicObjectSerializer(rpc_serializer.Serializer):
     that needs to accept or return IronicObjects as arguments or result values
     should pass this to its RpcProxy and RpcDispatcher objects.
     """
+
+    def _process_iterable(self, context, action_fn, values):
+        """Process an iterable, taking an action on each value.
+        :param:context: Request context
+        :param:action_fn: Action to take on each item in values
+        :param:values: Iterable container of things to take action on
+        :returns: A new container of the same type (except set) with
+                  items from values having had action applied.
+        """
+        iterable = values.__class__
+        if iterable == set:
+            # NOTE(danms): A set can't have an unhashable value inside, such as
+            # a dict. Convert sets to tuples, which is fine, since we can't
+            # send them over RPC anyway.
+            iterable = tuple
+        return iterable([action_fn(context, value) for value in values])
+
     def serialize_entity(self, context, entity):
-        if (hasattr(entity, 'obj_to_primitive') and
+        if isinstance(entity, (tuple, list, set)):
-            callable(entity.obj_to_primitive)):
+            entity = self._process_iterable(context, self.serialize_entity,
+                                            entity)
+        elif (hasattr(entity, 'obj_to_primitive') and
+                callable(entity.obj_to_primitive)):
             entity = entity.obj_to_primitive()
         return entity
 
     def deserialize_entity(self, context, entity):
         if isinstance(entity, dict) and 'ironic_object.name' in entity:
-            entity = IronicObject.obj_from_primitive(entity)
+            entity = IronicObject.obj_from_primitive(entity, context=context)
-            entity._context = context
+        elif isinstance(entity, (tuple, list, set)):
+            entity = self._process_iterable(context, self.deserialize_entity,
+                                            entity)
         return entity
+
+
+def obj_to_primitive(obj):
+    """Recursively turn an object into a python primitive.
+
+    An IronicObject becomes a dict, and anything that implements ObjectListBase
+    becomes a list.
+    """
+    if isinstance(obj, ObjectListBase):
+        return [obj_to_primitive(x) for x in obj]
+    elif isinstance(obj, IronicObject):
+        result = {}
+        for key, value in obj.iteritems():
+            result[key] = obj_to_primitive(value)
+        return result
+    else:
+        return obj

ironic/objects/utils.py

@@ -16,6 +16,7 @@
 
 import ast
 import datetime
+import iso8601
 import netaddr
 
 from ironic.openstack.common import timeutils
@@ -23,11 +24,25 @@ from ironic.openstack.common import timeutils
 
 def datetime_or_none(dt):
     """Validate a datetime or None value."""
-    if dt is None or isinstance(dt, datetime.datetime):
+    if dt is None:
-        return dt
+        return None
+    elif isinstance(dt, datetime.datetime):
+        if dt.utcoffset() is None:
+            # NOTE(danms): Legacy objects from sqlalchemy are stored in UTC,
+            # but are returned without a timezone attached.
+            # As a transitional aid, assume a tz-naive object is in UTC.
+            return dt.replace(tzinfo=iso8601.iso8601.Utc())
+        else:
+            return dt
     raise ValueError('A datetime.datetime is required here')
 
 
+def datetime_or_str_or_none(val):
+    if isinstance(val, basestring):
+        return timeutils.parse_isotime(val)
+    return datetime_or_none(val)
+
+
 def int_or_none(val):
     """Attempt to parse an integer value, or None."""
     if val is None:
@@ -67,6 +82,14 @@ def ip_or_none(version):
     return validator
 
 
+def nested_object_or_none(objclass):
+    def validator(val, objclass=objclass):
+        if val is None or isinstance(val, objclass):
+            return val
+        raise ValueError('An object of class %s is required here' % objclass)
+    return validator
+
+
 def dt_serializer(name):
     """Return a datetime serializer for a named attribute."""
     def serializer(self, name=name):
@@ -83,3 +106,12 @@ def dt_deserializer(instance, val):
         return None
     else:
         return timeutils.parse_isotime(val)
+
+
+def obj_serializer(name):
+    def serializer(self, name=name):
+        if getattr(self, name) is not None:
+            return getattr(self, name).obj_to_primitive()
+        else:
+            return None
+    return serializer

ironic/tests/base.py

@@ -26,6 +26,7 @@ inline callbacks.
 import eventlet
 eventlet.monkey_patch(os=False)
 
+import copy
 import os
 import shutil
 import sys
@@ -40,6 +41,7 @@ from oslo.config import cfg
 from ironic.db import migration
 
 from ironic.common import paths
+from ironic.objects import base as objects_base
 from ironic.openstack.common.db.sqlalchemy import session
 from ironic.openstack.common import log as logging
 from ironic.openstack.common import timeutils
@@ -182,6 +184,14 @@ class TestCase(testtools.TestCase):
                                     sqlite_clean_db=CONF.sqlite_clean_db)
         self.useFixture(_DB_CACHE)
 
+        # NOTE(danms): Make sure to reset us back to non-remote objects
+        # for each test to avoid interactions. Also, backup the object
+        # registry
+        objects_base.IronicObject.indirection_api = None
+        self._base_test_obj_backup = copy.copy(
+                objects_base.IronicObject._obj_classes)
+        self.addCleanup(self._restore_obj_registry)
+
         mox_fixture = self.useFixture(MoxStubout())
         self.mox = mox_fixture.mox
         self.stubs = mox_fixture.stubs
@@ -190,6 +200,9 @@ class TestCase(testtools.TestCase):
         self.policy = self.useFixture(policy_fixture.PolicyFixture())
         CONF.set_override('fatal_exception_format_errors', True)
 
+    def _restore_obj_registry(self):
+        objects_base.IronicObject._obj_classes = self._base_test_obj_backup
+
     def _clear_attrs(self):
         # Delete attributes that don't start with _ so they don't pin
         # memory around unnecessarily for the duration of the test

ironic/tests/objects/test_objects.py

@@ -15,6 +15,7 @@
 import contextlib
 import datetime
 import gettext
+import iso8601
 import netaddr
 
 gettext.install('ironic')
@@ -24,7 +25,7 @@ from ironic.objects import base
 from ironic.objects import utils
 from ironic.openstack.common import context
 from ironic.openstack.common import timeutils
-from ironic.tests.db import base as test_base
+from ironic.tests import base as test_base
 
 
 class MyObj(base.IronicObject):
@@ -34,7 +35,7 @@ class MyObj(base.IronicObject):
               'missing': str,
               }
 
-    def obj_load(self, attrname):
+    def obj_load_attr(self, attrname):
         setattr(self, attrname, 'loaded!')
 
     @base.remotable_classmethod
@@ -83,7 +84,7 @@ class MyObj2(object):
         pass
 
 
-class TestMetaclass(test_base.DbTestCase):
+class TestMetaclass(test_base.TestCase):
     def test_obj_tracking(self):
 
         class NewBaseClass(object):
@@ -115,13 +116,25 @@ class TestMetaclass(test_base.DbTestCase):
         self.assertEqual(expected, Test2._obj_classes)
 
 
-class TestUtils(test_base.DbTestCase):
+class TestUtils(test_base.TestCase):
     def test_datetime_or_none(self):
-        dt = datetime.datetime.now()
+        naive_dt = datetime.datetime.now()
+        dt = timeutils.parse_isotime(timeutils.isotime(naive_dt))
         self.assertEqual(utils.datetime_or_none(dt), dt)
+        self.assertEqual(utils.datetime_or_none(dt),
+                         naive_dt.replace(tzinfo=iso8601.iso8601.Utc(),
+                                          microsecond=0))
         self.assertEqual(utils.datetime_or_none(None), None)
         self.assertRaises(ValueError, utils.datetime_or_none, 'foo')
 
+    def test_datetime_or_str_or_none(self):
+        dts = timeutils.isotime()
+        dt = timeutils.parse_isotime(dts)
+        self.assertEqual(utils.datetime_or_str_or_none(dt), dt)
+        self.assertEqual(utils.datetime_or_str_or_none(None), None)
+        self.assertEqual(utils.datetime_or_str_or_none(dts), dt)
+        self.assertRaises(ValueError, utils.datetime_or_str_or_none, 'foo')
+
     def test_int_or_none(self):
         self.assertEqual(utils.int_or_none(1), 1)
         self.assertEqual(utils.int_or_none('1'), 1)
@@ -164,8 +177,33 @@ class TestUtils(test_base.DbTestCase):
         self.assertEqual(utils.dt_deserializer(None, None), None)
         self.assertRaises(ValueError, utils.dt_deserializer, None, 'foo')
 
+    def test_obj_to_primitive_list(self):
+        class MyList(base.ObjectListBase, base.IronicObject):
+            pass
+        mylist = MyList()
+        mylist.objects = [1, 2, 3]
+        self.assertEqual([1, 2, 3], base.obj_to_primitive(mylist))
 
-class _BaseTestCase(test_base.DbTestCase):
+    def test_obj_to_primitive_dict(self):
+        myobj = MyObj()
+        myobj.foo = 1
+        myobj.bar = 'foo'
+        self.assertEqual({'foo': 1, 'bar': 'foo'},
+                         base.obj_to_primitive(myobj))
+
+    def test_obj_to_primitive_recursive(self):
+        class MyList(base.ObjectListBase, base.IronicObject):
+            pass
+
+        mylist = MyList()
+        mylist.objects = [MyObj(), MyObj()]
+        for i, value in enumerate(mylist):
+            value.foo = i
+        self.assertEqual([{'foo': 0}, {'foo': 1}],
+                         base.obj_to_primitive(mylist))
+
+
+class _BaseTestCase(test_base.TestCase):
     def setUp(self):
         super(_BaseTestCase, self).setUp()
         self.remote_object_calls = list()
@@ -251,6 +289,19 @@ class _TestObject(object):
         obj = MyObj()
         self.assertEqual(obj.bar, 'loaded!')
 
+    def test_load_in_base(self):
+        class Foo(base.IronicObject):
+            fields = {'foobar': int}
+        obj = Foo()
+        # NOTE(danms): Can't use assertRaisesRegexp() because of py26
+        raised = False
+        try:
+            obj.foobar
+        except NotImplementedError as ex:
+            raised = True
+        self.assertTrue(raised)
+        self.assertTrue('foobar' in str(ex))
+
     def test_loaded_in_primitive(self):
         obj = MyObj()
         obj.foo = 1
@@ -370,6 +421,87 @@ class _TestObject(object):
                     }
         self.assertEqual(obj.obj_to_primitive(), expected)
 
+    def test_contains(self):
+        obj = MyObj()
+        self.assertFalse('foo' in obj)
+        obj.foo = 1
+        self.assertTrue('foo' in obj)
+        self.assertFalse('does_not_exist' in obj)
+
 
 class TestObject(_LocalTest, _TestObject):
     pass
+
+
+class TestObjectListBase(test_base.TestCase):
+    def test_list_like_operations(self):
+        class Foo(base.ObjectListBase, base.IronicObject):
+            pass
+
+        objlist = Foo()
+        objlist._context = 'foo'
+        objlist.objects = [1, 2, 3]
+        self.assertTrue(list(objlist), objlist.objects)
+        self.assertEqual(len(objlist), 3)
+        self.assertIn(2, objlist)
+        self.assertEqual(list(objlist[:1]), [1])
+        self.assertEqual(objlist[:1]._context, 'foo')
+        self.assertEqual(objlist[2], 3)
+        self.assertEqual(objlist.count(1), 1)
+        self.assertEqual(objlist.index(2), 1)
+
+    def test_serialization(self):
+        class Foo(base.ObjectListBase, base.IronicObject):
+            pass
+
+        class Bar(base.IronicObject):
+            fields = {'foo': str}
+
+        obj = Foo()
+        obj.objects = []
+        for i in 'abc':
+            bar = Bar()
+            bar.foo = i
+            obj.objects.append(bar)
+
+        obj2 = base.IronicObject.obj_from_primitive(obj.obj_to_primitive())
+        self.assertFalse(obj is obj2)
+        self.assertEqual([x.foo for x in obj],
+                         [y.foo for y in obj2])
+
+
+class TestObjectSerializer(test_base.TestCase):
+    def test_serialize_entity_primitive(self):
+        ser = base.IronicObjectSerializer()
+        for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
+            self.assertEqual(thing, ser.serialize_entity(None, thing))
+
+    def test_deserialize_entity_primitive(self):
+        ser = base.IronicObjectSerializer()
+        for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
+            self.assertEqual(thing, ser.deserialize_entity(None, thing))
+
+    def test_object_serialization(self):
+        ser = base.IronicObjectSerializer()
+        ctxt = context.get_admin_context()
+        obj = MyObj()
+        primitive = ser.serialize_entity(ctxt, obj)
+        self.assertTrue('ironic_object.name' in primitive)
+        obj2 = ser.deserialize_entity(ctxt, primitive)
+        self.assertTrue(isinstance(obj2, MyObj))
+        self.assertEqual(ctxt, obj2._context)
+
+    def test_object_serialization_iterables(self):
+        ser = base.IronicObjectSerializer()
+        ctxt = context.get_admin_context()
+        obj = MyObj()
+        for iterable in (list, tuple, set):
+            thing = iterable([obj])
+            primitive = ser.serialize_entity(ctxt, thing)
+            self.assertEqual(1, len(primitive))
+            for item in primitive:
+                self.assertFalse(isinstance(item, base.IronicObject))
+            thing2 = ser.deserialize_entity(ctxt, primitive)
+            self.assertEqual(1, len(thing2))
+            for item in thing2:
+                self.assertTrue(isinstance(item, MyObj))