From a5e5bd7cee6b8827dc89e9deee4cb2e36bca63a4 Mon Sep 17 00:00:00 2001 From: Devananda van der Veen Date: Wed, 17 Jul 2013 12:21:23 -0700 Subject: [PATCH] Import serialization and nesting from Nova Objects This change mostly merges the following commits from Nova: e91c3d141c957485dcb66c73e84b41b775e4268b f1c4b8e5f34eb6b5e70da6711750dcf05cea8c0a 65f6c536fecd3c788b2e0dfa9d66ecd24ca550e1 92a3190128547403dc603e5a40e377c6eb0c8025 68cb4d53385821c3ffdc40c299a77d11a7f98f27 Change-Id: I0a16f45674f5d14f458e2bb490d909a9086ea8b4 --- ironic/objects/base.py | 140 +++++++++++++++++++++++--- ironic/objects/utils.py | 36 ++++++- ironic/tests/base.py | 13 +++ ironic/tests/objects/test_objects.py | 144 +++++++++++++++++++++++++-- 4 files changed, 311 insertions(+), 22 deletions(-) diff --git a/ironic/objects/base.py b/ironic/objects/base.py index 0613cd2d99..ccc26fb6f2 100644 --- a/ironic/objects/base.py +++ b/ironic/objects/base.py @@ -37,10 +37,10 @@ def make_class_properties(cls): cls.fields.update(IronicObject.fields) for name, typefn in cls.fields.iteritems(): - def getter(self, name=name, typefn=typefn): + def getter(self, name=name): attrname = get_attrname(name) if not hasattr(self, attrname): - self.obj_load(name) + self.obj_load_attr(name) return getattr(self, attrname) def setter(self, value, name=name, typefn=typefn): @@ -175,9 +175,10 @@ class IronicObject(object): # by subclasses, but that is a special case. Objects inheriting from # other objects will not receive this merging of fields contents. fields = { - 'created_at': obj_utils.datetime_or_none, - 'updated_at': obj_utils.datetime_or_none, + 'created_at': obj_utils.datetime_or_str_or_none, + 'updated_at': obj_utils.datetime_or_str_or_none, } + obj_extra_fields = [] def __init__(self): self._changed_fields = set() @@ -216,7 +217,6 @@ class IronicObject(object): _attr_created_at_from_primitive = obj_utils.dt_deserializer _attr_updated_at_from_primitive = obj_utils.dt_deserializer - _attr_deleted_at_from_primitive = obj_utils.dt_deserializer def _attr_from_primitive(self, attribute, value): """Attribute deserialization dispatcher. @@ -231,7 +231,7 @@ class IronicObject(object): return value @classmethod - def obj_from_primitive(cls, primitive): + def obj_from_primitive(cls, primitive, context=None): """Simple base-case hydration. This calls self._attr_from_primitive() for each item in fields. @@ -247,6 +247,7 @@ class IronicObject(object): objdata = primitive['ironic_object.data'] objclass = cls.obj_class_from_name(objname, objver) self = objclass() + self._context = context for name in self.fields: if name in objdata: setattr(self, name, @@ -257,7 +258,6 @@ class IronicObject(object): _attr_created_at_to_primitive = obj_utils.dt_serializer('created_at') _attr_updated_at_to_primitive = obj_utils.dt_serializer('updated_at') - _attr_deleted_at_to_primitive = obj_utils.dt_serializer('deleted_at') def _attr_to_primitive(self, attribute): """Attribute serialization dispatcher. @@ -307,7 +307,7 @@ class IronicObject(object): raise NotImplementedError('Cannot save anything in the base class') def obj_what_changed(self): - """Returns a list of fields that have been modified.""" + """Returns a set of fields that have been modified.""" return self._changed_fields def obj_reset_changes(self, fields=None): @@ -326,8 +326,9 @@ class IronicObject(object): NOTE(danms): May be removed in the future. """ - for name in self.fields: - if hasattr(self, get_attrname(name)): + for name in self.fields.keys() + self.obj_extra_fields: + if (hasattr(self, get_attrname(name)) or + name in self.obj_extra_fields): yield name, getattr(self, name) items = lambda self: list(self.iteritems()) @@ -346,6 +347,13 @@ class IronicObject(object): """ setattr(self, name, value) + def __contains__(self, name): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + return hasattr(self, get_attrname(name)) + def get(self, key, value=None): """For backwards-compatibility with dict-based objects. @@ -353,6 +361,71 @@ class IronicObject(object): """ return self[key] + def update(self, updates): + """For backwards-compatibility with dict-base objects. + + NOTE(danms): May be removed in the future. + """ + for key, value in updates.items(): + self[key] = value + + +class ObjectListBase(object): + """Mixin class for lists of objects. + + This mixin class can be added as a base class for an object that + is implementing a list of objects. It adds a single field of 'objects', + which is the list store, and behaves like a list itself. It supports + serialization of the list of objects automatically. + """ + fields = { + 'objects': list, + } + + def __iter__(self): + """List iterator interface.""" + return iter(self.objects) + + def __len__(self): + """List length.""" + return len(self.objects) + + def __getitem__(self, index): + """List index access.""" + if isinstance(index, slice): + new_obj = self.__class__() + new_obj.objects = self.objects[index] + # NOTE(danms): We must be mixed in with an IronicObject! + new_obj.obj_reset_changes() + new_obj._context = self._context + return new_obj + return self.objects[index] + + def __contains__(self, value): + """List membership test.""" + return value in self.objects + + def count(self, value): + """List count of value occurrences.""" + return self.objects.count(value) + + def index(self, value): + """List index of value.""" + return self.objects.index(value) + + def _attr_objects_to_primitive(self): + """Serialization of object list.""" + return [x.obj_to_primitive() for x in self.objects] + + def _attr_objects_from_primitive(self, value): + """Deserialization of object list.""" + objects = [] + for entity in value: + obj = IronicObject.obj_from_primitive(entity, + context=self._context) + objects.append(obj) + return objects + class IronicObjectSerializer(rpc_serializer.Serializer): """A IronicObject-aware Serializer. @@ -362,14 +435,53 @@ class IronicObjectSerializer(rpc_serializer.Serializer): that needs to accept or return IronicObjects as arguments or result values should pass this to its RpcProxy and RpcDispatcher objects. """ + + def _process_iterable(self, context, action_fn, values): + """Process an iterable, taking an action on each value. + :param:context: Request context + :param:action_fn: Action to take on each item in values + :param:values: Iterable container of things to take action on + :returns: A new container of the same type (except set) with + items from values having had action applied. + """ + iterable = values.__class__ + if iterable == set: + # NOTE(danms): A set can't have an unhashable value inside, such as + # a dict. Convert sets to tuples, which is fine, since we can't + # send them over RPC anyway. + iterable = tuple + return iterable([action_fn(context, value) for value in values]) + def serialize_entity(self, context, entity): - if (hasattr(entity, 'obj_to_primitive') and - callable(entity.obj_to_primitive)): + if isinstance(entity, (tuple, list, set)): + entity = self._process_iterable(context, self.serialize_entity, + entity) + elif (hasattr(entity, 'obj_to_primitive') and + callable(entity.obj_to_primitive)): entity = entity.obj_to_primitive() return entity def deserialize_entity(self, context, entity): if isinstance(entity, dict) and 'ironic_object.name' in entity: - entity = IronicObject.obj_from_primitive(entity) - entity._context = context + entity = IronicObject.obj_from_primitive(entity, context=context) + elif isinstance(entity, (tuple, list, set)): + entity = self._process_iterable(context, self.deserialize_entity, + entity) return entity + + +def obj_to_primitive(obj): + """Recursively turn an object into a python primitive. + + An IronicObject becomes a dict, and anything that implements ObjectListBase + becomes a list. + """ + if isinstance(obj, ObjectListBase): + return [obj_to_primitive(x) for x in obj] + elif isinstance(obj, IronicObject): + result = {} + for key, value in obj.iteritems(): + result[key] = obj_to_primitive(value) + return result + else: + return obj diff --git a/ironic/objects/utils.py b/ironic/objects/utils.py index 89fdf6adf5..201ce91802 100644 --- a/ironic/objects/utils.py +++ b/ironic/objects/utils.py @@ -16,6 +16,7 @@ import ast import datetime +import iso8601 import netaddr from ironic.openstack.common import timeutils @@ -23,11 +24,25 @@ from ironic.openstack.common import timeutils def datetime_or_none(dt): """Validate a datetime or None value.""" - if dt is None or isinstance(dt, datetime.datetime): - return dt + if dt is None: + return None + elif isinstance(dt, datetime.datetime): + if dt.utcoffset() is None: + # NOTE(danms): Legacy objects from sqlalchemy are stored in UTC, + # but are returned without a timezone attached. + # As a transitional aid, assume a tz-naive object is in UTC. + return dt.replace(tzinfo=iso8601.iso8601.Utc()) + else: + return dt raise ValueError('A datetime.datetime is required here') +def datetime_or_str_or_none(val): + if isinstance(val, basestring): + return timeutils.parse_isotime(val) + return datetime_or_none(val) + + def int_or_none(val): """Attempt to parse an integer value, or None.""" if val is None: @@ -67,6 +82,14 @@ def ip_or_none(version): return validator +def nested_object_or_none(objclass): + def validator(val, objclass=objclass): + if val is None or isinstance(val, objclass): + return val + raise ValueError('An object of class %s is required here' % objclass) + return validator + + def dt_serializer(name): """Return a datetime serializer for a named attribute.""" def serializer(self, name=name): @@ -83,3 +106,12 @@ def dt_deserializer(instance, val): return None else: return timeutils.parse_isotime(val) + + +def obj_serializer(name): + def serializer(self, name=name): + if getattr(self, name) is not None: + return getattr(self, name).obj_to_primitive() + else: + return None + return serializer diff --git a/ironic/tests/base.py b/ironic/tests/base.py index 97775e62c6..139cf674ad 100644 --- a/ironic/tests/base.py +++ b/ironic/tests/base.py @@ -26,6 +26,7 @@ inline callbacks. import eventlet eventlet.monkey_patch(os=False) +import copy import os import shutil import sys @@ -40,6 +41,7 @@ from oslo.config import cfg from ironic.db import migration from ironic.common import paths +from ironic.objects import base as objects_base from ironic.openstack.common.db.sqlalchemy import session from ironic.openstack.common import log as logging from ironic.openstack.common import timeutils @@ -182,6 +184,14 @@ class TestCase(testtools.TestCase): sqlite_clean_db=CONF.sqlite_clean_db) self.useFixture(_DB_CACHE) + # NOTE(danms): Make sure to reset us back to non-remote objects + # for each test to avoid interactions. Also, backup the object + # registry + objects_base.IronicObject.indirection_api = None + self._base_test_obj_backup = copy.copy( + objects_base.IronicObject._obj_classes) + self.addCleanup(self._restore_obj_registry) + mox_fixture = self.useFixture(MoxStubout()) self.mox = mox_fixture.mox self.stubs = mox_fixture.stubs @@ -190,6 +200,9 @@ class TestCase(testtools.TestCase): self.policy = self.useFixture(policy_fixture.PolicyFixture()) CONF.set_override('fatal_exception_format_errors', True) + def _restore_obj_registry(self): + objects_base.IronicObject._obj_classes = self._base_test_obj_backup + def _clear_attrs(self): # Delete attributes that don't start with _ so they don't pin # memory around unnecessarily for the duration of the test diff --git a/ironic/tests/objects/test_objects.py b/ironic/tests/objects/test_objects.py index 102141c266..29708864a7 100644 --- a/ironic/tests/objects/test_objects.py +++ b/ironic/tests/objects/test_objects.py @@ -15,6 +15,7 @@ import contextlib import datetime import gettext +import iso8601 import netaddr gettext.install('ironic') @@ -24,7 +25,7 @@ from ironic.objects import base from ironic.objects import utils from ironic.openstack.common import context from ironic.openstack.common import timeutils -from ironic.tests.db import base as test_base +from ironic.tests import base as test_base class MyObj(base.IronicObject): @@ -34,7 +35,7 @@ class MyObj(base.IronicObject): 'missing': str, } - def obj_load(self, attrname): + def obj_load_attr(self, attrname): setattr(self, attrname, 'loaded!') @base.remotable_classmethod @@ -83,7 +84,7 @@ class MyObj2(object): pass -class TestMetaclass(test_base.DbTestCase): +class TestMetaclass(test_base.TestCase): def test_obj_tracking(self): class NewBaseClass(object): @@ -115,13 +116,25 @@ class TestMetaclass(test_base.DbTestCase): self.assertEqual(expected, Test2._obj_classes) -class TestUtils(test_base.DbTestCase): +class TestUtils(test_base.TestCase): def test_datetime_or_none(self): - dt = datetime.datetime.now() + naive_dt = datetime.datetime.now() + dt = timeutils.parse_isotime(timeutils.isotime(naive_dt)) self.assertEqual(utils.datetime_or_none(dt), dt) + self.assertEqual(utils.datetime_or_none(dt), + naive_dt.replace(tzinfo=iso8601.iso8601.Utc(), + microsecond=0)) self.assertEqual(utils.datetime_or_none(None), None) self.assertRaises(ValueError, utils.datetime_or_none, 'foo') + def test_datetime_or_str_or_none(self): + dts = timeutils.isotime() + dt = timeutils.parse_isotime(dts) + self.assertEqual(utils.datetime_or_str_or_none(dt), dt) + self.assertEqual(utils.datetime_or_str_or_none(None), None) + self.assertEqual(utils.datetime_or_str_or_none(dts), dt) + self.assertRaises(ValueError, utils.datetime_or_str_or_none, 'foo') + def test_int_or_none(self): self.assertEqual(utils.int_or_none(1), 1) self.assertEqual(utils.int_or_none('1'), 1) @@ -164,8 +177,33 @@ class TestUtils(test_base.DbTestCase): self.assertEqual(utils.dt_deserializer(None, None), None) self.assertRaises(ValueError, utils.dt_deserializer, None, 'foo') + def test_obj_to_primitive_list(self): + class MyList(base.ObjectListBase, base.IronicObject): + pass + mylist = MyList() + mylist.objects = [1, 2, 3] + self.assertEqual([1, 2, 3], base.obj_to_primitive(mylist)) -class _BaseTestCase(test_base.DbTestCase): + def test_obj_to_primitive_dict(self): + myobj = MyObj() + myobj.foo = 1 + myobj.bar = 'foo' + self.assertEqual({'foo': 1, 'bar': 'foo'}, + base.obj_to_primitive(myobj)) + + def test_obj_to_primitive_recursive(self): + class MyList(base.ObjectListBase, base.IronicObject): + pass + + mylist = MyList() + mylist.objects = [MyObj(), MyObj()] + for i, value in enumerate(mylist): + value.foo = i + self.assertEqual([{'foo': 0}, {'foo': 1}], + base.obj_to_primitive(mylist)) + + +class _BaseTestCase(test_base.TestCase): def setUp(self): super(_BaseTestCase, self).setUp() self.remote_object_calls = list() @@ -251,6 +289,19 @@ class _TestObject(object): obj = MyObj() self.assertEqual(obj.bar, 'loaded!') + def test_load_in_base(self): + class Foo(base.IronicObject): + fields = {'foobar': int} + obj = Foo() + # NOTE(danms): Can't use assertRaisesRegexp() because of py26 + raised = False + try: + obj.foobar + except NotImplementedError as ex: + raised = True + self.assertTrue(raised) + self.assertTrue('foobar' in str(ex)) + def test_loaded_in_primitive(self): obj = MyObj() obj.foo = 1 @@ -370,6 +421,87 @@ class _TestObject(object): } self.assertEqual(obj.obj_to_primitive(), expected) + def test_contains(self): + obj = MyObj() + self.assertFalse('foo' in obj) + obj.foo = 1 + self.assertTrue('foo' in obj) + self.assertFalse('does_not_exist' in obj) + class TestObject(_LocalTest, _TestObject): pass + + +class TestObjectListBase(test_base.TestCase): + def test_list_like_operations(self): + class Foo(base.ObjectListBase, base.IronicObject): + pass + + objlist = Foo() + objlist._context = 'foo' + objlist.objects = [1, 2, 3] + self.assertTrue(list(objlist), objlist.objects) + self.assertEqual(len(objlist), 3) + self.assertIn(2, objlist) + self.assertEqual(list(objlist[:1]), [1]) + self.assertEqual(objlist[:1]._context, 'foo') + self.assertEqual(objlist[2], 3) + self.assertEqual(objlist.count(1), 1) + self.assertEqual(objlist.index(2), 1) + + def test_serialization(self): + class Foo(base.ObjectListBase, base.IronicObject): + pass + + class Bar(base.IronicObject): + fields = {'foo': str} + + obj = Foo() + obj.objects = [] + for i in 'abc': + bar = Bar() + bar.foo = i + obj.objects.append(bar) + + obj2 = base.IronicObject.obj_from_primitive(obj.obj_to_primitive()) + self.assertFalse(obj is obj2) + self.assertEqual([x.foo for x in obj], + [y.foo for y in obj2]) + + +class TestObjectSerializer(test_base.TestCase): + def test_serialize_entity_primitive(self): + ser = base.IronicObjectSerializer() + for thing in (1, 'foo', [1, 2], {'foo': 'bar'}): + self.assertEqual(thing, ser.serialize_entity(None, thing)) + + def test_deserialize_entity_primitive(self): + ser = base.IronicObjectSerializer() + for thing in (1, 'foo', [1, 2], {'foo': 'bar'}): + self.assertEqual(thing, ser.deserialize_entity(None, thing)) + + def test_object_serialization(self): + ser = base.IronicObjectSerializer() + ctxt = context.get_admin_context() + obj = MyObj() + primitive = ser.serialize_entity(ctxt, obj) + self.assertTrue('ironic_object.name' in primitive) + obj2 = ser.deserialize_entity(ctxt, primitive) + self.assertTrue(isinstance(obj2, MyObj)) + self.assertEqual(ctxt, obj2._context) + + def test_object_serialization_iterables(self): + ser = base.IronicObjectSerializer() + ctxt = context.get_admin_context() + obj = MyObj() + for iterable in (list, tuple, set): + thing = iterable([obj]) + primitive = ser.serialize_entity(ctxt, thing) + self.assertEqual(1, len(primitive)) + for item in primitive: + self.assertFalse(isinstance(item, base.IronicObject)) + thing2 = ser.deserialize_entity(ctxt, primitive) + self.assertEqual(1, len(thing2)) + for item in thing2: + self.assertTrue(isinstance(item, MyObj))