Import serialization and nesting from Nova Objects
This change mostly merges the following commits from Nova: e91c3d141c957485dcb66c73e84b41b775e4268b f1c4b8e5f34eb6b5e70da6711750dcf05cea8c0a 65f6c536fecd3c788b2e0dfa9d66ecd24ca550e1 92a3190128547403dc603e5a40e377c6eb0c8025 68cb4d53385821c3ffdc40c299a77d11a7f98f27 Change-Id: I0a16f45674f5d14f458e2bb490d909a9086ea8b4
This commit is contained in:
Devananda van der Veen
@@ -37,10 +37,10 @@ def make_class_properties(cls):
|
||||
cls.fields.update(IronicObject.fields)
|
||||
for name, typefn in cls.fields.iteritems():
|
||||
|
||||
def getter(self, name=name, typefn=typefn):
|
||||
def getter(self, name=name):
|
||||
attrname = get_attrname(name)
|
||||
if not hasattr(self, attrname):
|
||||
self.obj_load(name)
|
||||
self.obj_load_attr(name)
|
||||
return getattr(self, attrname)
|
||||
|
||||
def setter(self, value, name=name, typefn=typefn):
|
||||
@@ -175,9 +175,10 @@ class IronicObject(object):
|
||||
# by subclasses, but that is a special case. Objects inheriting from
|
||||
# other objects will not receive this merging of fields contents.
|
||||
fields = {
|
||||
'created_at': obj_utils.datetime_or_none,
|
||||
'updated_at': obj_utils.datetime_or_none,
|
||||
'created_at': obj_utils.datetime_or_str_or_none,
|
||||
'updated_at': obj_utils.datetime_or_str_or_none,
|
||||
}
|
||||
obj_extra_fields = []
|
||||
|
||||
def __init__(self):
|
||||
self._changed_fields = set()
|
||||
@@ -216,7 +217,6 @@ class IronicObject(object):
|
||||
|
||||
_attr_created_at_from_primitive = obj_utils.dt_deserializer
|
||||
_attr_updated_at_from_primitive = obj_utils.dt_deserializer
|
||||
_attr_deleted_at_from_primitive = obj_utils.dt_deserializer
|
||||
|
||||
def _attr_from_primitive(self, attribute, value):
|
||||
"""Attribute deserialization dispatcher.
|
||||
@@ -231,7 +231,7 @@ class IronicObject(object):
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def obj_from_primitive(cls, primitive):
|
||||
def obj_from_primitive(cls, primitive, context=None):
|
||||
"""Simple base-case hydration.
|
||||
|
||||
This calls self._attr_from_primitive() for each item in fields.
|
||||
@@ -247,6 +247,7 @@ class IronicObject(object):
|
||||
objdata = primitive['ironic_object.data']
|
||||
objclass = cls.obj_class_from_name(objname, objver)
|
||||
self = objclass()
|
||||
self._context = context
|
||||
for name in self.fields:
|
||||
if name in objdata:
|
||||
setattr(self, name,
|
||||
@@ -257,7 +258,6 @@ class IronicObject(object):
|
||||
|
||||
_attr_created_at_to_primitive = obj_utils.dt_serializer('created_at')
|
||||
_attr_updated_at_to_primitive = obj_utils.dt_serializer('updated_at')
|
||||
_attr_deleted_at_to_primitive = obj_utils.dt_serializer('deleted_at')
|
||||
|
||||
def _attr_to_primitive(self, attribute):
|
||||
"""Attribute serialization dispatcher.
|
||||
@@ -307,7 +307,7 @@ class IronicObject(object):
|
||||
raise NotImplementedError('Cannot save anything in the base class')
|
||||
|
||||
def obj_what_changed(self):
|
||||
"""Returns a list of fields that have been modified."""
|
||||
"""Returns a set of fields that have been modified."""
|
||||
return self._changed_fields
|
||||
|
||||
def obj_reset_changes(self, fields=None):
|
||||
@@ -326,8 +326,9 @@ class IronicObject(object):
|
||||
|
||||
NOTE(danms): May be removed in the future.
|
||||
"""
|
||||
for name in self.fields:
|
||||
if hasattr(self, get_attrname(name)):
|
||||
for name in self.fields.keys() + self.obj_extra_fields:
|
||||
if (hasattr(self, get_attrname(name)) or
|
||||
name in self.obj_extra_fields):
|
||||
yield name, getattr(self, name)
|
||||
|
||||
items = lambda self: list(self.iteritems())
|
||||
@@ -346,6 +347,13 @@ class IronicObject(object):
|
||||
"""
|
||||
setattr(self, name, value)
|
||||
|
||||
def __contains__(self, name):
|
||||
"""For backwards-compatibility with dict-based objects.
|
||||
|
||||
NOTE(danms): May be removed in the future.
|
||||
"""
|
||||
return hasattr(self, get_attrname(name))
|
||||
|
||||
def get(self, key, value=None):
|
||||
"""For backwards-compatibility with dict-based objects.
|
||||
|
||||
@@ -353,6 +361,71 @@ class IronicObject(object):
|
||||
"""
|
||||
return self[key]
|
||||
|
||||
def update(self, updates):
|
||||
"""For backwards-compatibility with dict-base objects.
|
||||
|
||||
NOTE(danms): May be removed in the future.
|
||||
"""
|
||||
for key, value in updates.items():
|
||||
self[key] = value
|
||||
|
||||
|
||||
class ObjectListBase(object):
|
||||
"""Mixin class for lists of objects.
|
||||
|
||||
This mixin class can be added as a base class for an object that
|
||||
is implementing a list of objects. It adds a single field of 'objects',
|
||||
which is the list store, and behaves like a list itself. It supports
|
||||
serialization of the list of objects automatically.
|
||||
"""
|
||||
fields = {
|
||||
'objects': list,
|
||||
}
|
||||
|
||||
def __iter__(self):
|
||||
"""List iterator interface."""
|
||||
return iter(self.objects)
|
||||
|
||||
def __len__(self):
|
||||
"""List length."""
|
||||
return len(self.objects)
|
||||
|
||||
def __getitem__(self, index):
|
||||
"""List index access."""
|
||||
if isinstance(index, slice):
|
||||
new_obj = self.__class__()
|
||||
new_obj.objects = self.objects[index]
|
||||
# NOTE(danms): We must be mixed in with an IronicObject!
|
||||
new_obj.obj_reset_changes()
|
||||
new_obj._context = self._context
|
||||
return new_obj
|
||||
return self.objects[index]
|
||||
|
||||
def __contains__(self, value):
|
||||
"""List membership test."""
|
||||
return value in self.objects
|
||||
|
||||
def count(self, value):
|
||||
"""List count of value occurrences."""
|
||||
return self.objects.count(value)
|
||||
|
||||
def index(self, value):
|
||||
"""List index of value."""
|
||||
return self.objects.index(value)
|
||||
|
||||
def _attr_objects_to_primitive(self):
|
||||
"""Serialization of object list."""
|
||||
return [x.obj_to_primitive() for x in self.objects]
|
||||
|
||||
def _attr_objects_from_primitive(self, value):
|
||||
"""Deserialization of object list."""
|
||||
objects = []
|
||||
for entity in value:
|
||||
obj = IronicObject.obj_from_primitive(entity,
|
||||
context=self._context)
|
||||
objects.append(obj)
|
||||
return objects
|
||||
|
||||
|
||||
class IronicObjectSerializer(rpc_serializer.Serializer):
|
||||
"""A IronicObject-aware Serializer.
|
||||
@@ -362,14 +435,53 @@ class IronicObjectSerializer(rpc_serializer.Serializer):
|
||||
that needs to accept or return IronicObjects as arguments or result values
|
||||
should pass this to its RpcProxy and RpcDispatcher objects.
|
||||
"""
|
||||
|
||||
def _process_iterable(self, context, action_fn, values):
|
||||
"""Process an iterable, taking an action on each value.
|
||||
:param:context: Request context
|
||||
:param:action_fn: Action to take on each item in values
|
||||
:param:values: Iterable container of things to take action on
|
||||
:returns: A new container of the same type (except set) with
|
||||
items from values having had action applied.
|
||||
"""
|
||||
iterable = values.__class__
|
||||
if iterable == set:
|
||||
# NOTE(danms): A set can't have an unhashable value inside, such as
|
||||
# a dict. Convert sets to tuples, which is fine, since we can't
|
||||
# send them over RPC anyway.
|
||||
iterable = tuple
|
||||
return iterable([action_fn(context, value) for value in values])
|
||||
|
||||
def serialize_entity(self, context, entity):
|
||||
if (hasattr(entity, 'obj_to_primitive') and
|
||||
callable(entity.obj_to_primitive)):
|
||||
if isinstance(entity, (tuple, list, set)):
|
||||
entity = self._process_iterable(context, self.serialize_entity,
|
||||
entity)
|
||||
elif (hasattr(entity, 'obj_to_primitive') and
|
||||
callable(entity.obj_to_primitive)):
|
||||
entity = entity.obj_to_primitive()
|
||||
return entity
|
||||
|
||||
def deserialize_entity(self, context, entity):
|
||||
if isinstance(entity, dict) and 'ironic_object.name' in entity:
|
||||
entity = IronicObject.obj_from_primitive(entity)
|
||||
entity._context = context
|
||||
entity = IronicObject.obj_from_primitive(entity, context=context)
|
||||
elif isinstance(entity, (tuple, list, set)):
|
||||
entity = self._process_iterable(context, self.deserialize_entity,
|
||||
entity)
|
||||
return entity
|
||||
|
||||
|
||||
def obj_to_primitive(obj):
|
||||
"""Recursively turn an object into a python primitive.
|
||||
|
||||
An IronicObject becomes a dict, and anything that implements ObjectListBase
|
||||
becomes a list.
|
||||
"""
|
||||
if isinstance(obj, ObjectListBase):
|
||||
return [obj_to_primitive(x) for x in obj]
|
||||
elif isinstance(obj, IronicObject):
|
||||
result = {}
|
||||
for key, value in obj.iteritems():
|
||||
result[key] = obj_to_primitive(value)
|
||||
return result
|
||||
else:
|
||||
return obj
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import ast
|
||||
import datetime
|
||||
import iso8601
|
||||
import netaddr
|
||||
|
||||
from ironic.openstack.common import timeutils
|
||||
@@ -23,11 +24,25 @@ from ironic.openstack.common import timeutils
|
||||
|
||||
def datetime_or_none(dt):
|
||||
"""Validate a datetime or None value."""
|
||||
if dt is None or isinstance(dt, datetime.datetime):
|
||||
return dt
|
||||
if dt is None:
|
||||
return None
|
||||
elif isinstance(dt, datetime.datetime):
|
||||
if dt.utcoffset() is None:
|
||||
# NOTE(danms): Legacy objects from sqlalchemy are stored in UTC,
|
||||
# but are returned without a timezone attached.
|
||||
# As a transitional aid, assume a tz-naive object is in UTC.
|
||||
return dt.replace(tzinfo=iso8601.iso8601.Utc())
|
||||
else:
|
||||
return dt
|
||||
raise ValueError('A datetime.datetime is required here')
|
||||
|
||||
|
||||
def datetime_or_str_or_none(val):
|
||||
if isinstance(val, basestring):
|
||||
return timeutils.parse_isotime(val)
|
||||
return datetime_or_none(val)
|
||||
|
||||
|
||||
def int_or_none(val):
|
||||
"""Attempt to parse an integer value, or None."""
|
||||
if val is None:
|
||||
@@ -67,6 +82,14 @@ def ip_or_none(version):
|
||||
return validator
|
||||
|
||||
|
||||
def nested_object_or_none(objclass):
|
||||
def validator(val, objclass=objclass):
|
||||
if val is None or isinstance(val, objclass):
|
||||
return val
|
||||
raise ValueError('An object of class %s is required here' % objclass)
|
||||
return validator
|
||||
|
||||
|
||||
def dt_serializer(name):
|
||||
"""Return a datetime serializer for a named attribute."""
|
||||
def serializer(self, name=name):
|
||||
@@ -83,3 +106,12 @@ def dt_deserializer(instance, val):
|
||||
return None
|
||||
else:
|
||||
return timeutils.parse_isotime(val)
|
||||
|
||||
|
||||
def obj_serializer(name):
|
||||
def serializer(self, name=name):
|
||||
if getattr(self, name) is not None:
|
||||
return getattr(self, name).obj_to_primitive()
|
||||
else:
|
||||
return None
|
||||
return serializer
|
||||
|
||||
@@ -26,6 +26,7 @@ inline callbacks.
|
||||
import eventlet
|
||||
eventlet.monkey_patch(os=False)
|
||||
|
||||
import copy
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
@@ -40,6 +41,7 @@ from oslo.config import cfg
|
||||
from ironic.db import migration
|
||||
|
||||
from ironic.common import paths
|
||||
from ironic.objects import base as objects_base
|
||||
from ironic.openstack.common.db.sqlalchemy import session
|
||||
from ironic.openstack.common import log as logging
|
||||
from ironic.openstack.common import timeutils
|
||||
@@ -182,6 +184,14 @@ class TestCase(testtools.TestCase):
|
||||
sqlite_clean_db=CONF.sqlite_clean_db)
|
||||
self.useFixture(_DB_CACHE)
|
||||
|
||||
# NOTE(danms): Make sure to reset us back to non-remote objects
|
||||
# for each test to avoid interactions. Also, backup the object
|
||||
# registry
|
||||
objects_base.IronicObject.indirection_api = None
|
||||
self._base_test_obj_backup = copy.copy(
|
||||
objects_base.IronicObject._obj_classes)
|
||||
self.addCleanup(self._restore_obj_registry)
|
||||
|
||||
mox_fixture = self.useFixture(MoxStubout())
|
||||
self.mox = mox_fixture.mox
|
||||
self.stubs = mox_fixture.stubs
|
||||
@@ -190,6 +200,9 @@ class TestCase(testtools.TestCase):
|
||||
self.policy = self.useFixture(policy_fixture.PolicyFixture())
|
||||
CONF.set_override('fatal_exception_format_errors', True)
|
||||
|
||||
def _restore_obj_registry(self):
|
||||
objects_base.IronicObject._obj_classes = self._base_test_obj_backup
|
||||
|
||||
def _clear_attrs(self):
|
||||
# Delete attributes that don't start with _ so they don't pin
|
||||
# memory around unnecessarily for the duration of the test
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import contextlib
|
||||
import datetime
|
||||
import gettext
|
||||
import iso8601
|
||||
import netaddr
|
||||
|
||||
gettext.install('ironic')
|
||||
@@ -24,7 +25,7 @@ from ironic.objects import base
|
||||
from ironic.objects import utils
|
||||
from ironic.openstack.common import context
|
||||
from ironic.openstack.common import timeutils
|
||||
from ironic.tests.db import base as test_base
|
||||
from ironic.tests import base as test_base
|
||||
|
||||
|
||||
class MyObj(base.IronicObject):
|
||||
@@ -34,7 +35,7 @@ class MyObj(base.IronicObject):
|
||||
'missing': str,
|
||||
}
|
||||
|
||||
def obj_load(self, attrname):
|
||||
def obj_load_attr(self, attrname):
|
||||
setattr(self, attrname, 'loaded!')
|
||||
|
||||
@base.remotable_classmethod
|
||||
@@ -83,7 +84,7 @@ class MyObj2(object):
|
||||
pass
|
||||
|
||||
|
||||
class TestMetaclass(test_base.DbTestCase):
|
||||
class TestMetaclass(test_base.TestCase):
|
||||
def test_obj_tracking(self):
|
||||
|
||||
class NewBaseClass(object):
|
||||
@@ -115,13 +116,25 @@ class TestMetaclass(test_base.DbTestCase):
|
||||
self.assertEqual(expected, Test2._obj_classes)
|
||||
|
||||
|
||||
class TestUtils(test_base.DbTestCase):
|
||||
class TestUtils(test_base.TestCase):
|
||||
def test_datetime_or_none(self):
|
||||
dt = datetime.datetime.now()
|
||||
naive_dt = datetime.datetime.now()
|
||||
dt = timeutils.parse_isotime(timeutils.isotime(naive_dt))
|
||||
self.assertEqual(utils.datetime_or_none(dt), dt)
|
||||
self.assertEqual(utils.datetime_or_none(dt),
|
||||
naive_dt.replace(tzinfo=iso8601.iso8601.Utc(),
|
||||
microsecond=0))
|
||||
self.assertEqual(utils.datetime_or_none(None), None)
|
||||
self.assertRaises(ValueError, utils.datetime_or_none, 'foo')
|
||||
|
||||
def test_datetime_or_str_or_none(self):
|
||||
dts = timeutils.isotime()
|
||||
dt = timeutils.parse_isotime(dts)
|
||||
self.assertEqual(utils.datetime_or_str_or_none(dt), dt)
|
||||
self.assertEqual(utils.datetime_or_str_or_none(None), None)
|
||||
self.assertEqual(utils.datetime_or_str_or_none(dts), dt)
|
||||
self.assertRaises(ValueError, utils.datetime_or_str_or_none, 'foo')
|
||||
|
||||
def test_int_or_none(self):
|
||||
self.assertEqual(utils.int_or_none(1), 1)
|
||||
self.assertEqual(utils.int_or_none('1'), 1)
|
||||
@@ -164,8 +177,33 @@ class TestUtils(test_base.DbTestCase):
|
||||
self.assertEqual(utils.dt_deserializer(None, None), None)
|
||||
self.assertRaises(ValueError, utils.dt_deserializer, None, 'foo')
|
||||
|
||||
def test_obj_to_primitive_list(self):
|
||||
class MyList(base.ObjectListBase, base.IronicObject):
|
||||
pass
|
||||
mylist = MyList()
|
||||
mylist.objects = [1, 2, 3]
|
||||
self.assertEqual([1, 2, 3], base.obj_to_primitive(mylist))
|
||||
|
||||
class _BaseTestCase(test_base.DbTestCase):
|
||||
def test_obj_to_primitive_dict(self):
|
||||
myobj = MyObj()
|
||||
myobj.foo = 1
|
||||
myobj.bar = 'foo'
|
||||
self.assertEqual({'foo': 1, 'bar': 'foo'},
|
||||
base.obj_to_primitive(myobj))
|
||||
|
||||
def test_obj_to_primitive_recursive(self):
|
||||
class MyList(base.ObjectListBase, base.IronicObject):
|
||||
pass
|
||||
|
||||
mylist = MyList()
|
||||
mylist.objects = [MyObj(), MyObj()]
|
||||
for i, value in enumerate(mylist):
|
||||
value.foo = i
|
||||
self.assertEqual([{'foo': 0}, {'foo': 1}],
|
||||
base.obj_to_primitive(mylist))
|
||||
|
||||
|
||||
class _BaseTestCase(test_base.TestCase):
|
||||
def setUp(self):
|
||||
super(_BaseTestCase, self).setUp()
|
||||
self.remote_object_calls = list()
|
||||
@@ -251,6 +289,19 @@ class _TestObject(object):
|
||||
obj = MyObj()
|
||||
self.assertEqual(obj.bar, 'loaded!')
|
||||
|
||||
def test_load_in_base(self):
|
||||
class Foo(base.IronicObject):
|
||||
fields = {'foobar': int}
|
||||
obj = Foo()
|
||||
# NOTE(danms): Can't use assertRaisesRegexp() because of py26
|
||||
raised = False
|
||||
try:
|
||||
obj.foobar
|
||||
except NotImplementedError as ex:
|
||||
raised = True
|
||||
self.assertTrue(raised)
|
||||
self.assertTrue('foobar' in str(ex))
|
||||
|
||||
def test_loaded_in_primitive(self):
|
||||
obj = MyObj()
|
||||
obj.foo = 1
|
||||
@@ -370,6 +421,87 @@ class _TestObject(object):
|
||||
}
|
||||
self.assertEqual(obj.obj_to_primitive(), expected)
|
||||
|
||||
def test_contains(self):
|
||||
obj = MyObj()
|
||||
self.assertFalse('foo' in obj)
|
||||
obj.foo = 1
|
||||
self.assertTrue('foo' in obj)
|
||||
self.assertFalse('does_not_exist' in obj)
|
||||
|
||||
|
||||
class TestObject(_LocalTest, _TestObject):
|
||||
pass
|
||||
|
||||
|
||||
class TestObjectListBase(test_base.TestCase):
|
||||
def test_list_like_operations(self):
|
||||
class Foo(base.ObjectListBase, base.IronicObject):
|
||||
pass
|
||||
|
||||
objlist = Foo()
|
||||
objlist._context = 'foo'
|
||||
objlist.objects = [1, 2, 3]
|
||||
self.assertTrue(list(objlist), objlist.objects)
|
||||
self.assertEqual(len(objlist), 3)
|
||||
self.assertIn(2, objlist)
|
||||
self.assertEqual(list(objlist[:1]), [1])
|
||||
self.assertEqual(objlist[:1]._context, 'foo')
|
||||
self.assertEqual(objlist[2], 3)
|
||||
self.assertEqual(objlist.count(1), 1)
|
||||
self.assertEqual(objlist.index(2), 1)
|
||||
|
||||
def test_serialization(self):
|
||||
class Foo(base.ObjectListBase, base.IronicObject):
|
||||
pass
|
||||
|
||||
class Bar(base.IronicObject):
|
||||
fields = {'foo': str}
|
||||
|
||||
obj = Foo()
|
||||
obj.objects = []
|
||||
for i in 'abc':
|
||||
bar = Bar()
|
||||
bar.foo = i
|
||||
obj.objects.append(bar)
|
||||
|
||||
obj2 = base.IronicObject.obj_from_primitive(obj.obj_to_primitive())
|
||||
self.assertFalse(obj is obj2)
|
||||
self.assertEqual([x.foo for x in obj],
|
||||
[y.foo for y in obj2])
|
||||
|
||||
|
||||
class TestObjectSerializer(test_base.TestCase):
|
||||
def test_serialize_entity_primitive(self):
|
||||
ser = base.IronicObjectSerializer()
|
||||
for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
|
||||
self.assertEqual(thing, ser.serialize_entity(None, thing))
|
||||
|
||||
def test_deserialize_entity_primitive(self):
|
||||
ser = base.IronicObjectSerializer()
|
||||
for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
|
||||
self.assertEqual(thing, ser.deserialize_entity(None, thing))
|
||||
|
||||
def test_object_serialization(self):
|
||||
ser = base.IronicObjectSerializer()
|
||||
ctxt = context.get_admin_context()
|
||||
obj = MyObj()
|
||||
primitive = ser.serialize_entity(ctxt, obj)
|
||||
self.assertTrue('ironic_object.name' in primitive)
|
||||
obj2 = ser.deserialize_entity(ctxt, primitive)
|
||||
self.assertTrue(isinstance(obj2, MyObj))
|
||||
self.assertEqual(ctxt, obj2._context)
|
||||
|
||||
def test_object_serialization_iterables(self):
|
||||
ser = base.IronicObjectSerializer()
|
||||
ctxt = context.get_admin_context()
|
||||
obj = MyObj()
|
||||
for iterable in (list, tuple, set):
|
||||
thing = iterable([obj])
|
||||
primitive = ser.serialize_entity(ctxt, thing)
|
||||
self.assertEqual(1, len(primitive))
|
||||
for item in primitive:
|
||||
self.assertFalse(isinstance(item, base.IronicObject))
|
||||
thing2 = ser.deserialize_entity(ctxt, primitive)
|
||||
self.assertEqual(1, len(thing2))
|
||||
for item in thing2:
|
||||
self.assertTrue(isinstance(item, MyObj))
|
||||
|
||||
Reference in New Issue
Block a user