Cinder objects base

This patch is to insert the objects code base, currently based
on nova.objects.  It will be later based on
oslo_versionedobjects once it is available.  This is done
to make progress on changing cinder internals to use objects,
since nova.objects and oslo.versionedobjects are similar.

Inserted objects code base into cinder and modified cinder services
to use object-aware serializer.

Change-Id: I42a6cc43da19ca29db4cfb33cbf794afed18f758
Implements: blueprint cinder-objects
This commit is contained in:
Thang Pham 2014-11-10 17:02:02 -05:00
parent d9cd6d008c
commit 2b4c8e79ec
18 changed files with 2887 additions and 7 deletions

View File

@ -23,6 +23,8 @@ eventlet.monkey_patch()
import sys
import warnings
from cinder import objects
warnings.simplefilter('once', DeprecationWarning)
from oslo_config import cfg
@ -43,6 +45,7 @@ CONF = cfg.CONF
def main():
objects.register_all()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup("cinder")

View File

@ -76,6 +76,7 @@ from cinder import db
from cinder.db import migration as db_migration
from cinder.db.sqlalchemy import api as db_api
from cinder.i18n import _
from cinder.objects import base as objects_base
from cinder.openstack.common import log as logging
from cinder import rpc
from cinder import utils
@ -271,7 +272,9 @@ class VolumeCommands(object):
if not rpc.initialized():
rpc.init(CONF)
target = messaging.Target(topic=CONF.volume_topic)
self._client = rpc.get_client(target)
serializer = objects_base.CinderObjectSerializer()
self._client = rpc.get_client(target, serializer=serializer)
return self._client
@args('volume_id',

View File

@ -21,6 +21,8 @@ import os
import eventlet
from cinder import objects
if os.name == 'nt':
# eventlet monkey patching the os module causes subprocess.Popen to fail
# on Windows when using pipes due to missing non-blocking IO support.
@ -54,6 +56,7 @@ CONF = cfg.CONF
def main():
objects.register_all()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup("cinder")

View File

@ -657,6 +657,30 @@ class EvaluatorParseException(Exception):
message = _("Error during evaluator parsing: %(reason)s")
class ObjectActionError(CinderException):
msg_fmt = _('Object action %(action)s failed because: %(reason)s')
class ObjectFieldInvalid(CinderException):
msg_fmt = _('Field %(field)s of %(objname)s is not an instance of Field')
class UnsupportedObjectError(CinderException):
msg_fmt = _('Unsupported object type %(objtype)s')
class OrphanedObjectError(CinderException):
msg_fmt = _('Cannot call %(method)s on orphaned %(objtype)s object')
class IncompatibleObjectVersion(CinderException):
msg_fmt = _('Version %(objver)s of %(objname)s is not supported')
class ReadOnlyFieldError(CinderException):
msg_fmt = _('Cannot modify readonly field %(field)s')
# Driver specific exceptions
# Coraid
class CoraidException(VolumeDriverException):

View File

@ -29,7 +29,8 @@ Guidelines for writing new hacking checks
"""
UNDERSCORE_IMPORT_FILES = []
# NOTE(thangp): Ignore N323 pep8 error caused by importing cinder objects
UNDERSCORE_IMPORT_FILES = ['./cinder/objects/__init__.py']
translated_log = re.compile(
r"(.)*LOG\.(audit|error|info|warn|warning|critical|exception)"

View File

@ -0,0 +1,27 @@
# Copyright 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# NOTE(comstud): You may scratch your head as you see code that imports
# this module and then accesses attributes for objects such as Instance,
# etc, yet you do not see these attributes in here. Never fear, there is
# a little bit of magic. When objects are registered, an attribute is set
# on this module automatically, pointing to the newest/latest version of
# the object.
def register_all():
# NOTE(danms): You must make sure your object gets imported in this
# function in order for it to be registered by services that may
# need to receive it via RPC.
pass

867
cinder/objects/base.py Normal file
View File

@ -0,0 +1,867 @@
# Copyright 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Cinder common internal object model"""
import collections
import contextlib
import copy
import datetime
import functools
import traceback
import netaddr
from oslo import messaging
from oslo_utils import timeutils
import six
from cinder import context
from cinder import exception
from cinder.i18n import _, _LE
from cinder import objects
from cinder.objects import fields
from cinder.openstack.common import log as logging
from cinder.openstack.common import versionutils
from cinder import utils
LOG = logging.getLogger('object')
class NotSpecifiedSentinel(object):
pass
def get_attrname(name):
"""Return the mangled name of the attribute's underlying storage."""
return '_' + name
def make_class_properties(cls):
# NOTE(danms/comstud): Inherit fields from super classes.
# mro() returns the current class first and returns 'object' last, so
# those can be skipped. Also be careful to not overwrite any fields
# that already exist. And make sure each cls has its own copy of
# fields and that it is not sharing the dict with a super class.
cls.fields = dict(cls.fields)
for supercls in cls.mro()[1:-1]:
if not hasattr(supercls, 'fields'):
continue
for name, field in supercls.fields.items():
if name not in cls.fields:
cls.fields[name] = field
for name, field in cls.fields.iteritems():
if not isinstance(field, fields.Field):
raise exception.ObjectFieldInvalid(
field=name, objname=cls.obj_name())
def getter(self, name=name):
attrname = get_attrname(name)
if not hasattr(self, attrname):
self.obj_load_attr(name)
return getattr(self, attrname)
def setter(self, value, name=name, field=field):
attrname = get_attrname(name)
field_value = field.coerce(self, name, value)
if field.read_only and hasattr(self, attrname):
# Note(yjiang5): _from_db_object() may iterate
# every field and write, no exception in such situation.
if getattr(self, attrname) != field_value:
raise exception.ReadOnlyFieldError(field=name)
else:
return
self._changed_fields.add(name)
try:
return setattr(self, attrname, field_value)
except Exception:
attr = "%s.%s" % (self.obj_name(), name)
LOG.exception(_LE('Error setting %(attr)s'), {'attr': attr})
raise
setattr(cls, name, property(getter, setter))
class CinderObjectMetaclass(type):
"""Metaclass that allows tracking of object classes."""
# NOTE(danms): This is what controls whether object operations are
# remoted. If this is not None, use it to remote things over RPC.
indirection_api = None
def __init__(cls, names, bases, dict_):
if not hasattr(cls, '_obj_classes'):
# This means this is a base class using the metaclass. I.e.,
# the 'CinderObject' class.
cls._obj_classes = collections.defaultdict(list)
return
def _vers_tuple(obj):
return tuple([int(x) for x in obj.VERSION.split(".")])
# Add the subclass to CinderObject._obj_classes. If the
# same version already exists, replace it. Otherwise,
# keep the list with newest version first.
make_class_properties(cls)
obj_name = cls.obj_name()
for i, obj in enumerate(cls._obj_classes[obj_name]):
if cls.VERSION == obj.VERSION:
cls._obj_classes[obj_name][i] = cls
# Update cinder.objects with this newer class.
setattr(objects, obj_name, cls)
break
if _vers_tuple(cls) > _vers_tuple(obj):
# Insert before.
cls._obj_classes[obj_name].insert(i, cls)
if i == 0:
# Later version than we've seen before. Update
# cinder.objects.
setattr(objects, obj_name, cls)
break
else:
cls._obj_classes[obj_name].append(cls)
# Either this is the first time we've seen the object or it's
# an older version than anything we'e seen. Update cinder.objects
# only if it's the first time we've seen this object name.
if not hasattr(objects, obj_name):
setattr(objects, obj_name, cls)
# These are decorators that mark an object's method as remotable.
# If the metaclass is configured to forward object methods to an
# indirection service, these will result in making an RPC call
# instead of directly calling the implementation in the object. Instead,
# the object implementation on the remote end will perform the
# requested action and the result will be returned here.
def remotable_classmethod(fn):
"""Decorator for remotable classmethods."""
@functools.wraps(fn)
def wrapper(cls, context, *args, **kwargs):
if CinderObject.indirection_api:
result = CinderObject.indirection_api.object_class_action(
context, cls.obj_name(), fn.__name__, cls.VERSION,
args, kwargs)
else:
result = fn(cls, context, *args, **kwargs)
if isinstance(result, CinderObject):
result._context = context
return result
# NOTE(danms): Make this discoverable
wrapper.remotable = True
wrapper.original_fn = fn
return classmethod(wrapper)
# See comment above for remotable_classmethod()
#
# Note that this will use either the provided context, or the one
# stashed in the object. If neither are present, the object is
# "orphaned" and remotable methods cannot be called.
def remotable(fn):
"""Decorator for remotable object methods."""
@functools.wraps(fn)
def wrapper(self, *args, **kwargs):
ctxt = self._context
try:
if isinstance(args[0], (context.RequestContext)):
ctxt = args[0]
args = args[1:]
except IndexError:
pass
if ctxt is None:
raise exception.OrphanedObjectError(method=fn.__name__,
objtype=self.obj_name())
# Force this to be set if it wasn't before.
self._context = ctxt
if CinderObject.indirection_api:
updates, result = CinderObject.indirection_api.object_action(
ctxt, self, fn.__name__, args, kwargs)
for key, value in updates.iteritems():
if key in self.fields:
field = self.fields[key]
# NOTE(ndipanov): Since CinderObjectSerializer will have
# deserialized any object fields into objects already,
# we do not try to deserialize them again here.
if isinstance(value, CinderObject):
self[key] = value
else:
self[key] = field.from_primitive(self, key, value)
self.obj_reset_changes()
self._changed_fields = set(updates.get('obj_what_changed', []))
return result
else:
return fn(self, ctxt, *args, **kwargs)
wrapper.remotable = True
wrapper.original_fn = fn
return wrapper
@six.add_metaclass(CinderObjectMetaclass)
class CinderObject(object):
"""Base class and object factory.
This forms the base of all objects that can be remoted or instantiated
via RPC. Simply defining a class that inherits from this base class
will make it remotely instantiatable. Objects should implement the
necessary "get" classmethod routines as well as "save" object methods
as appropriate.
"""
# Object versioning rules
#
# Each service has its set of objects, each with a version attached. When
# a client attempts to call an object method, the server checks to see if
# the version of that object matches (in a compatible way) its object
# implementation. If so, cool, and if not, fail.
#
# This version is allowed to have three parts, X.Y.Z, where the .Z element
# is reserved for stable branch backports. The .Z is ignored for the
# purposes of triggering a backport, which means anything changed under
# a .Z must be additive and non-destructive such that a node that knows
# about X.Y can consider X.Y.Z equivalent.
VERSION = '1.0'
# The fields present in this object as key:field pairs. For example:
#
# fields = { 'foo': fields.IntegerField(),
# 'bar': fields.StringField(),
# }
fields = {}
obj_extra_fields = []
# Table of sub-object versioning information
#
# This contains a list of version mappings, by the field name of
# the subobject. The mappings must be in order of oldest to
# newest, and are tuples of (my_version, subobject_version). A
# request to backport this object to $my_version will cause the
# subobject to be backported to $subobject_version.
#
# obj_relationships = {
# 'subobject1': [('1.2', '1.1'), ('1.4', '1.2')],
# 'subobject2': [('1.2', '1.0')],
# }
#
# In the above example:
#
# - If we are asked to backport our object to version 1.3,
# subobject1 will be backported to version 1.1, since it was
# bumped to version 1.2 when our version was 1.4.
# - If we are asked to backport our object to version 1.5,
# no changes will be made to subobject1 or subobject2, since
# they have not changed since version 1.4.
# - If we are asked to backlevel our object to version 1.1, we
# will remove both subobject1 and subobject2 from the primitive,
# since they were not added until version 1.2.
obj_relationships = {}
def __init__(self, context=None, **kwargs):
self._changed_fields = set()
self._context = context
for key in kwargs.keys():
setattr(self, key, kwargs[key])
def __repr__(self):
return '%s(%s)' % (
self.obj_name(),
','.join(['%s=%s' % (name,
(self.obj_attr_is_set(name) and
field.stringify(getattr(self, name)) or
'<?>'))
for name, field in sorted(self.fields.items())]))
@classmethod
def obj_name(cls):
"""Return a canonical name for this object.
The canonical name will be used over the wire for remote hydration.
"""
return cls.__name__
@classmethod
def obj_class_from_name(cls, objname, objver):
"""Returns a class from the registry based on a name and version."""
if objname not in cls._obj_classes:
LOG.error(_LE('Unable to instantiate unregistered object type '
'%(objtype)s'), dict(objtype=objname))
raise exception.UnsupportedObjectError(objtype=objname)
# NOTE(comstud): If there's not an exact match, return the highest
# compatible version. The objects stored in the class are sorted
# such that highest version is first, so only set compatible_match
# once below.
compatible_match = None
for objclass in cls._obj_classes[objname]:
if objclass.VERSION == objver:
return objclass
if (not compatible_match and
versionutils.is_compatible(objver, objclass.VERSION)):
compatible_match = objclass
if compatible_match:
return compatible_match
# As mentioned above, latest version is always first in the list.
latest_ver = cls._obj_classes[objname][0].VERSION
raise exception.IncompatibleObjectVersion(objname=objname,
objver=objver,
supported=latest_ver)
@classmethod
def _obj_from_primitive(cls, context, objver, primitive):
self = cls()
self._context = context
self.VERSION = objver
objdata = primitive['cinder_object.data']
changes = primitive.get('cinder_object.changes', [])
for name, field in self.fields.items():
if name in objdata:
setattr(self, name, field.from_primitive(self, name,
objdata[name]))
self._changed_fields = set([x for x in changes if x in self.fields])
return self
@classmethod
def obj_from_primitive(cls, primitive, context=None):
"""Object field-by-field hydration."""
if primitive['cinder_object.namespace'] != 'cinder':
# NOTE(danms): We don't do anything with this now, but it's
# there for "the future"
raise exception.UnsupportedObjectError(
objtype='%s.%s' % (primitive['cinder_object.namespace'],
primitive['cinder_object.name']))
objname = primitive['cinder_object.name']
objver = primitive['cinder_object.version']
objclass = cls.obj_class_from_name(objname, objver)
return objclass._obj_from_primitive(context, objver, primitive)
def __deepcopy__(self, memo):
"""Efficiently make a deep copy of this object."""
# NOTE(danms): A naive deepcopy would copy more than we need,
# and since we have knowledge of the volatile bits of the
# object, we can be smarter here. Also, nested entities within
# some objects may be uncopyable, so we can avoid those sorts
# of issues by copying only our field data.
nobj = self.__class__()
nobj._context = self._context
for name in self.fields:
if self.obj_attr_is_set(name):
nval = copy.deepcopy(getattr(self, name), memo)
setattr(nobj, name, nval)
nobj._changed_fields = set(self._changed_fields)
return nobj
def obj_clone(self):
"""Create a copy."""
return copy.deepcopy(self)
def _obj_make_obj_compatible(self, primitive, target_version, field):
"""Backlevel a sub-object based on our versioning rules.
This is responsible for backporting objects contained within
this object's primitive according to a set of rules we
maintain about version dependencies between objects. This
requires that the obj_relationships table in this object is
correct and up-to-date.
:param:primitive: The primitive version of this object
:param:target_version: The version string requested for this object
:param:field: The name of the field in this object containing the
sub-object to be backported
"""
def _do_backport(to_version):
obj = getattr(self, field)
if not obj:
return
if isinstance(obj, CinderObject):
obj.obj_make_compatible(
primitive[field]['cinder_object.data'],
to_version)
primitive[field]['cinder_object.version'] = to_version
elif isinstance(obj, list):
for i, element in enumerate(obj):
element.obj_make_compatible(
primitive[field][i]['cinder_object.data'],
to_version)
primitive[field][i]['cinder_object.version'] = to_version
target_version = utils.convert_version_to_tuple(target_version)
for index, versions in enumerate(self.obj_relationships[field]):
my_version, child_version = versions
my_version = utils.convert_version_to_tuple(my_version)
if target_version < my_version:
if index == 0:
# We're backporting to a version from before this
# subobject was added: delete it from the primitive.
del primitive[field]
else:
# We're in the gap between index-1 and index, so
# backport to the older version
last_child_version = \
self.obj_relationships[field][index - 1][1]
_do_backport(last_child_version)
return
elif target_version == my_version:
# This is the first mapping that satisfies the
# target_version request: backport the object.
_do_backport(child_version)
return
def obj_make_compatible(self, primitive, target_version):
"""Make an object representation compatible with a target version.
This is responsible for taking the primitive representation of
an object and making it suitable for the given target_version.
This may mean converting the format of object attributes, removing
attributes that have been added since the target version, etc. In
general:
- If a new version of an object adds a field, this routine
should remove it for older versions.
- If a new version changed or restricted the format of a field, this
should convert it back to something a client knowing only of the
older version will tolerate.
- If an object that this object depends on is bumped, then this
object should also take a version bump. Then, this routine should
backlevel the dependent object (by calling its obj_make_compatible())
if the requested version of this object is older than the version
where the new dependent object was added.
:param:primitive: The result of self.obj_to_primitive()
:param:target_version: The version string requested by the recipient
of the object
:raises: cinder.exception.UnsupportedObjectError if conversion
is not possible for some reason
"""
for key, field in self.fields.items():
if not isinstance(field, (fields.ObjectField,
fields.ListOfObjectsField)):
continue
if not self.obj_attr_is_set(key):
continue
if key not in self.obj_relationships:
# NOTE(danms): This is really a coding error and shouldn't
# happen unless we miss something
raise exception.ObjectActionError(
action='obj_make_compatible',
reason='No rule for %s' % key)
self._obj_make_obj_compatible(primitive, target_version, key)
def obj_to_primitive(self, target_version=None):
"""Simple base-case dehydration.
This calls to_primitive() for each item in fields.
"""
primitive = dict()
for name, field in self.fields.items():
if self.obj_attr_is_set(name):
primitive[name] = field.to_primitive(self, name,
getattr(self, name))
if target_version:
self.obj_make_compatible(primitive, target_version)
obj = {'cinder_object.name': self.obj_name(),
'cinder_object.namespace': 'cinder',
'cinder_object.version': target_version or self.VERSION,
'cinder_object.data': primitive}
if self.obj_what_changed():
obj['cinder_object.changes'] = list(self.obj_what_changed())
return obj
def obj_set_defaults(self, *attrs):
if not attrs:
attrs = [name for name, field in self.fields.items()
if field.default != fields.UnspecifiedDefault]
for attr in attrs:
default = self.fields[attr].default
if default is fields.UnspecifiedDefault:
raise exception.ObjectActionError(
action='set_defaults',
reason='No default set for field %s' % attr)
setattr(self, attr, default)
def obj_load_attr(self, attrname):
"""Load an additional attribute from the real object."""
raise NotImplementedError(
_("Cannot load '%s' in the base class") % attrname)
def save(self, context):
"""Save the changed fields back to the store.
This is optional for subclasses, but is presented here in the base
class for consistency among those that do.
"""
raise NotImplementedError('Cannot save anything in the base class')
def obj_what_changed(self):
"""Returns a set of fields that have been modified."""
changes = set(self._changed_fields)
for field in self.fields:
if (self.obj_attr_is_set(field) and
isinstance(getattr(self, field), CinderObject) and
getattr(self, field).obj_what_changed()):
changes.add(field)
return changes
def obj_get_changes(self):
"""Returns a dict of changed fields and their new values."""
changes = {}
for key in self.obj_what_changed():
changes[key] = getattr(self, key)
return changes
def obj_reset_changes(self, fields=None):
"""Reset the list of fields that have been changed.
Note that this is NOT "revert to previous values"
"""
if fields:
self._changed_fields -= set(fields)
else:
self._changed_fields.clear()
def obj_attr_is_set(self, attrname):
"""Test object to see if attrname is present.
Returns True if the named attribute has a value set, or
False if not. Raises AttributeError if attrname is not
a valid attribute for this object.
"""
if attrname not in self.obj_fields:
raise AttributeError(
_("%(objname)s object has no attribute '%(attrname)s'") %
{'objname': self.obj_name(), 'attrname': attrname})
return hasattr(self, get_attrname(attrname))
@property
def obj_fields(self):
return self.fields.keys() + self.obj_extra_fields
class CinderObjectDictCompat(object):
"""Mix-in to provide dictionary key access compat
If an object needs to support attribute access using
dictionary items instead of object attributes, inherit
from this class. This should only be used as a temporary
measure until all callers are converted to use modern
attribute access.
NOTE(berrange) This class will eventually be deleted.
"""
# dictish syntactic sugar
def iteritems(self):
"""For backwards-compatibility with dict-based objects.
NOTE(danms): May be removed in the future.
"""
for name in self.obj_fields:
if (self.obj_attr_is_set(name) or
name in self.obj_extra_fields):
yield name, getattr(self, name)
items = lambda self: list(self.iteritems())
def __getitem__(self, name):
"""For backwards-compatibility with dict-based objects.
NOTE(danms): May be removed in the future.
"""
return getattr(self, name)
def __setitem__(self, name, value):
"""For backwards-compatibility with dict-based objects.
NOTE(danms): May be removed in the future.
"""
setattr(self, name, value)
def __contains__(self, name):
"""For backwards-compatibility with dict-based objects.
NOTE(danms): May be removed in the future.
"""
try:
return self.obj_attr_is_set(name)
except AttributeError:
return False
def get(self, key, value=NotSpecifiedSentinel):
"""For backwards-compatibility with dict-based objects.
NOTE(danms): May be removed in the future.
"""
if key not in self.obj_fields:
raise AttributeError("'%s' object has no attribute '%s'" % (
self.__class__, key))
if value != NotSpecifiedSentinel and not self.obj_attr_is_set(key):
return value
else:
return getattr(self, key)
def update(self, updates):
"""For backwards-compatibility with dict-base objects.
NOTE(danms): May be removed in the future.
"""
for key, value in updates.items():
setattr(self, key, value)
class CinderPersistentObject(object):
"""Mixin class for Persistent objects.
This adds the fields that we use in common for all persistent objects.
"""
fields = {
'created_at': fields.DateTimeField(nullable=True),
'updated_at': fields.DateTimeField(nullable=True),
'deleted_at': fields.DateTimeField(nullable=True),
'deleted': fields.BooleanField(default=False),
}
@contextlib.contextmanager
def obj_as_admin(self):
"""Context manager to make an object call as an admin.
This temporarily modifies the context embedded in an object to
be elevated() and restores it after the call completes. Example
usage:
with obj.obj_as_admin():
obj.save()
"""
if self._context is None:
raise exception.OrphanedObjectError(method='obj_as_admin',
objtype=self.obj_name())
original_context = self._context
self._context = self._context.elevated()
try:
yield
finally:
self._context = original_context
class ObjectListBase(object):
"""Mixin class for lists of objects.
This mixin class can be added as a base class for an object that
is implementing a list of objects. It adds a single field of 'objects',
which is the list store, and behaves like a list itself. It supports
serialization of the list of objects automatically.
"""
fields = {
'objects': fields.ListOfObjectsField('CinderObject'),
}
# This is a dictionary of my_version:child_version mappings so that
# we can support backleveling our contents based on the version
# requested of the list object.
child_versions = {}
def __init__(self, *args, **kwargs):
super(ObjectListBase, self).__init__(*args, **kwargs)
if 'objects' not in kwargs:
self.objects = []
self._changed_fields.discard('objects')
def __iter__(self):
"""List iterator interface."""
return iter(self.objects)
def __len__(self):
"""List length."""
return len(self.objects)
def __getitem__(self, index):
"""List index access."""
if isinstance(index, slice):
new_obj = self.__class__()
new_obj.objects = self.objects[index]
# NOTE(danms): We must be mixed in with a CinderObject!
new_obj.obj_reset_changes()
new_obj._context = self._context
return new_obj
return self.objects[index]
def __contains__(self, value):
"""List membership test."""
return value in self.objects
def count(self, value):
"""List count of value occurrences."""
return self.objects.count(value)
def index(self, value):
"""List index of value."""
return self.objects.index(value)
def sort(self, cmp=None, key=None, reverse=False):
self.objects.sort(cmp=cmp, key=key, reverse=reverse)
def obj_make_compatible(self, primitive, target_version):
primitives = primitive['objects']
child_target_version = self.child_versions.get(target_version, '1.0')
for index, item in enumerate(self.objects):
self.objects[index].obj_make_compatible(
primitives[index]['cinder_object.data'],
child_target_version)
primitives[index]['cinder_object.version'] = child_target_version
def obj_what_changed(self):
changes = set(self._changed_fields)
for child in self.objects:
if child.obj_what_changed():
changes.add('objects')
return changes
class CinderObjectSerializer(messaging.NoOpSerializer):
"""A CinderObject-aware Serializer.
This implements the Oslo Serializer interface and provides the
ability to serialize and deserialize CinderObject entities. Any service
that needs to accept or return CinderObjects as arguments or result values
should pass this to its RPCClient and RPCServer objects.
"""
def _process_object(self, context, objprim):
try:
objinst = CinderObject.obj_from_primitive(objprim, context=context)
except exception.IncompatibleObjectVersion:
objver = objprim['cinder_object.version']
if objver.count('.') == 2:
# NOTE(danms): For our purposes, the .z part of the version
# should be safe to accept without requiring a backport
objprim['cinder_object.version'] = \
'.'.join(objver.split('.')[:2])
return self._process_object(context, objprim)
raise
return objinst
def _process_iterable(self, context, action_fn, values):
"""Process an iterable, taking an action on each value.
:param:context: Request context
:param:action_fn: Action to take on each item in values
:param:values: Iterable container of things to take action on
:returns: A new container of the same type (except set) with
items from values having had action applied.
"""
iterable = values.__class__
if issubclass(iterable, dict):
return iterable(**{k: action_fn(context, v)
for k, v in six.iteritems(values)})
else:
# NOTE(danms): A set can't have an unhashable value inside, such as
# a dict. Convert sets to tuples, which is fine, since we can't
# send them over RPC anyway.
if iterable == set:
iterable = tuple
return iterable([action_fn(context, value) for value in values])
def serialize_entity(self, context, entity):
if isinstance(entity, (tuple, list, set, dict)):
entity = self._process_iterable(context, self.serialize_entity,
entity)
elif (hasattr(entity, 'obj_to_primitive') and
callable(entity.obj_to_primitive)):
entity = entity.obj_to_primitive()
return entity
def deserialize_entity(self, context, entity):
if isinstance(entity, dict) and 'cinder_object.name' in entity:
entity = self._process_object(context, entity)
elif isinstance(entity, (tuple, list, set, dict)):
entity = self._process_iterable(context, self.deserialize_entity,
entity)
return entity
def obj_to_primitive(obj):
"""Recursively turn an object into a python primitive.
A CinderObject becomes a dict, and anything that implements ObjectListBase
becomes a list.
"""
if isinstance(obj, ObjectListBase):
return [obj_to_primitive(x) for x in obj]
elif isinstance(obj, CinderObject):
result = {}
for key in obj.obj_fields:
if obj.obj_attr_is_set(key) or key in obj.obj_extra_fields:
result[key] = obj_to_primitive(getattr(obj, key))
return result
elif isinstance(obj, netaddr.IPAddress):
return str(obj)
elif isinstance(obj, netaddr.IPNetwork):
return str(obj)
else:
return obj
def obj_make_list(context, list_obj, item_cls, db_list, **extra_args):
"""Construct an object list from a list of primitives.
This calls item_cls._from_db_object() on each item of db_list, and
adds the resulting object to list_obj.
:param:context: Request contextr
:param:list_obj: An ObjectListBase object
:param:item_cls: The CinderObject class of the objects within the list
:param:db_list: The list of primitives to convert to objects
:param:extra_args: Extra arguments to pass to _from_db_object()
:returns: list_obj
"""
list_obj.objects = []
for db_item in db_list:
item = item_cls._from_db_object(context, item_cls(), db_item,
**extra_args)
list_obj.objects.append(item)
list_obj._context = context
list_obj.obj_reset_changes()
return list_obj
def serialize_args(fn):
"""Decorator that will do the arguments serialization before remoting."""
def wrapper(obj, *args, **kwargs):
for kw in kwargs:
value_arg = kwargs.get(kw)
if kw == 'exc_val' and value_arg:
kwargs[kw] = str(value_arg)
elif kw == 'exc_tb' and (
not isinstance(value_arg, six.string_types) and value_arg):
kwargs[kw] = ''.join(traceback.format_tb(value_arg))
elif isinstance(value_arg, datetime.datetime):
kwargs[kw] = timeutils.isotime(value_arg)
if hasattr(fn, '__call__'):
return fn(obj, *args, **kwargs)
# NOTE(danms): We wrap a descriptor, so use that protocol
return fn.__get__(None, obj)(*args, **kwargs)
# NOTE(danms): Make this discoverable
wrapper.remotable = getattr(fn, 'remotable', False)
wrapper.original_fn = fn
return (functools.wraps(fn)(wrapper) if hasattr(fn, '__call__')
else classmethod(wrapper))

607
cinder/objects/fields.py Normal file
View File

@ -0,0 +1,607 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import datetime
import iso8601
import netaddr
from oslo_utils import timeutils
import six
from cinder.i18n import _
class KeyTypeError(TypeError):
def __init__(self, expected, value):
super(KeyTypeError, self).__init__(
_('Key %(key)s must be of type %(expected)s not %(actual)s'
) % {'key': repr(value),
'expected': expected.__name__,
'actual': value.__class__.__name__,
})
class ElementTypeError(TypeError):
def __init__(self, expected, key, value):
super(ElementTypeError, self).__init__(
_('Element %(key)s:%(val)s must be of type %(expected)s'
' not %(actual)s'
) % {'key': key,
'val': repr(value),
'expected': expected,
'actual': value.__class__.__name__,
})
@six.add_metaclass(abc.ABCMeta)
class AbstractFieldType(object):
@abc.abstractmethod
def coerce(self, obj, attr, value):
"""This is called to coerce (if possible) a value on assignment.
This method should convert the value given into the designated type,
or throw an exception if this is not possible.
:param:obj: The CinderObject on which an attribute is being set
:param:attr: The name of the attribute being set
:param:value: The value being set
:returns: A properly-typed value
"""
pass
@abc.abstractmethod
def from_primitive(self, obj, attr, value):
"""This is called to deserialize a value.
This method should deserialize a value from the form given by
to_primitive() to the designated type.
:param:obj: The CinderObject on which the value is to be set
:param:attr: The name of the attribute which will hold the value
:param:value: The serialized form of the value
:returns: The natural form of the value
"""
pass
@abc.abstractmethod
def to_primitive(self, obj, attr, value):
"""This is called to serialize a value.
This method should serialize a value to the form expected by
from_primitive().
:param:obj: The CinderObject on which the value is set
:param:attr: The name of the attribute holding the value
:param:value: The natural form of the value
:returns: The serialized form of the value
"""
pass
@abc.abstractmethod
def describe(self):
"""Returns a string describing the type of the field."""
pass
@abc.abstractmethod
def stringify(self, value):
"""Returns a short stringified version of a value."""
pass
class FieldType(AbstractFieldType):
@staticmethod
def coerce(obj, attr, value):
return value
@staticmethod
def from_primitive(obj, attr, value):
return value
@staticmethod
def to_primitive(obj, attr, value):
return value
def describe(self):
return self.__class__.__name__
def stringify(self, value):
return str(value)
class UnspecifiedDefault(object):
pass
class Field(object):
def __init__(self, field_type, nullable=False,
default=UnspecifiedDefault, read_only=False):
self._type = field_type
self._nullable = nullable
self._default = default
self._read_only = read_only
def __repr__(self):
args = {
'nullable': self._nullable,
'default': self._default,
}
return '%s(%s)' % (self._type.__class__.__name__,
','.join(['%s=%s' % (k, v)
for k, v in args.items()]))
@property
def nullable(self):
return self._nullable
@property
def default(self):
return self._default
@property
def read_only(self):
return self._read_only
def _null(self, obj, attr):
if self.nullable:
return None
elif self._default != UnspecifiedDefault:
# NOTE(danms): We coerce the default value each time the field
# is set to None as our contract states that we'll let the type
# examine the object and attribute name at that time.
return self._type.coerce(obj, attr, self._default)
else:
raise ValueError(_("Field `%s' cannot be None") % attr)
def coerce(self, obj, attr, value):
"""Coerce a value to a suitable type.
This is called any time you set a value on an object, like:
foo.myint = 1
and is responsible for making sure that the value (1 here) is of
the proper type, or can be sanely converted.
This also handles the potentially nullable or defaultable
nature of the field and calls the coerce() method on a
FieldType to actually do the coercion.
:param:obj: The object being acted upon
:param:attr: The name of the attribute/field being set
:param:value: The value being set
:returns: The properly-typed value
"""
if value is None:
return self._null(obj, attr)
else:
return self._type.coerce(obj, attr, value)
def from_primitive(self, obj, attr, value):
"""Deserialize a value from primitive form.
This is responsible for deserializing a value from primitive
into regular form. It calls the from_primitive() method on a
FieldType to do the actual deserialization.
:param:obj: The object being acted upon
:param:attr: The name of the attribute/field being deserialized
:param:value: The value to be deserialized
:returns: The deserialized value
"""
if value is None:
return None
else:
return self._type.from_primitive(obj, attr, value)
def to_primitive(self, obj, attr, value):
"""Serialize a value to primitive form.
This is responsible for serializing a value to primitive
form. It calls to_primitive() on a FieldType to do the actual
serialization.
:param:obj: The object being acted upon
:param:attr: The name of the attribute/field being serialized
:param:value: The value to be serialized
:returns: The serialized value
"""
if value is None:
return None
else:
return self._type.to_primitive(obj, attr, value)
def describe(self):
"""Return a short string describing the type of this field."""
name = self._type.describe()
prefix = self.nullable and 'Nullable' or ''
return prefix + name
def stringify(self, value):
if value is None:
return 'None'
else:
return self._type.stringify(value)
class String(FieldType):
@staticmethod
def coerce(obj, attr, value):
# FIXME(danms): We should really try to avoid the need to do this
if isinstance(value, (six.string_types, int, long, float,
datetime.datetime)):
return unicode(value)
else:
raise ValueError(_('A string is required here, not %s') %
value.__class__.__name__)
@staticmethod
def stringify(value):
return "'%s'" % value
class UUID(FieldType):
@staticmethod
def coerce(obj, attr, value):
# FIXME(danms): We should actually verify the UUIDness here
return six.text_type(value)
class Integer(FieldType):
@staticmethod
def coerce(obj, attr, value):
return int(value)
class Float(FieldType):
def coerce(self, obj, attr, value):
return float(value)
class Boolean(FieldType):
@staticmethod
def coerce(obj, attr, value):
return bool(value)
class DateTime(FieldType):
@staticmethod
def coerce(obj, attr, value):
if isinstance(value, six.string_types):
# NOTE(danms): Being tolerant of isotime strings here will help us
# during our objects transition
value = timeutils.parse_isotime(value)
elif not isinstance(value, datetime.datetime):
raise ValueError(_('A datetime.datetime is required here'))
if value.utcoffset() is None:
# NOTE(danms): Legacy objects from sqlalchemy are stored in UTC,
# but are returned without a timezone attached.
# As a transitional aid, assume a tz-naive object is in UTC.
value = value.replace(tzinfo=iso8601.iso8601.Utc())
return value
def from_primitive(self, obj, attr, value):
return self.coerce(obj, attr, timeutils.parse_isotime(value))
@staticmethod
def to_primitive(obj, attr, value):
return timeutils.isotime(value)
@staticmethod
def stringify(value):
return timeutils.isotime(value)
class IPAddress(FieldType):
@staticmethod
def coerce(obj, attr, value):
try:
return netaddr.IPAddress(value)
except netaddr.AddrFormatError as e:
raise ValueError(six.text_type(e))
def from_primitive(self, obj, attr, value):
return self.coerce(obj, attr, value)
@staticmethod
def to_primitive(obj, attr, value):
return six.text_type(value)
class IPV4Address(IPAddress):
@staticmethod
def coerce(obj, attr, value):
result = IPAddress.coerce(obj, attr, value)
if result.version != 4:
raise ValueError(_('Network "%s" is not valid') % value)
return result
class IPV6Address(IPAddress):
@staticmethod
def coerce(obj, attr, value):
result = IPAddress.coerce(obj, attr, value)
if result.version != 6:
raise ValueError(_('Network "%s" is not valid') % value)
return result
class IPV4AndV6Address(IPAddress):
@staticmethod
def coerce(obj, attr, value):
result = IPAddress.coerce(obj, attr, value)
if result.version != 4 and result.version != 6:
raise ValueError(_('Network "%s" is not valid') % value)
return result
class IPNetwork(IPAddress):
@staticmethod
def coerce(obj, attr, value):
try:
return netaddr.IPNetwork(value)
except netaddr.AddrFormatError as e:
raise ValueError(six.text_type(e))
class IPV4Network(IPNetwork):
@staticmethod
def coerce(obj, attr, value):
try:
return netaddr.IPNetwork(value, version=4)
except netaddr.AddrFormatError as e:
raise ValueError(six.text_type(e))
class IPV6Network(IPNetwork):
@staticmethod
def coerce(obj, attr, value):
try:
return netaddr.IPNetwork(value, version=6)
except netaddr.AddrFormatError as e:
raise ValueError(six.text_type(e))
class CompoundFieldType(FieldType):
def __init__(self, element_type, **field_args):
self._element_type = Field(element_type, **field_args)
class List(CompoundFieldType):
def coerce(self, obj, attr, value):
if not isinstance(value, list):
raise ValueError(_('A list is required here'))
for index, element in enumerate(list(value)):
value[index] = self._element_type.coerce(
obj, '%s[%i]' % (attr, index), element)
return value
def to_primitive(self, obj, attr, value):
return [self._element_type.to_primitive(obj, attr, x) for x in value]
def from_primitive(self, obj, attr, value):
return [self._element_type.from_primitive(obj, attr, x) for x in value]
def stringify(self, value):
return '[%s]' % (
','.join([self._element_type.stringify(x) for x in value]))
class Dict(CompoundFieldType):
def coerce(self, obj, attr, value):
if not isinstance(value, dict):
raise ValueError(_('A dict is required here'))
for key, element in value.items():
if not isinstance(key, six.string_types):
# NOTE(guohliu) In order to keep compatibility with python3
# we need to use six.string_types rather than basestring here,
# since six.string_types is a tuple, so we need to pass the
# real type in.
raise KeyTypeError(six.string_types[0], key)
value[key] = self._element_type.coerce(
obj, '%s["%s"]' % (attr, key), element)
return value
def to_primitive(self, obj, attr, value):
primitive = {}
for key, element in value.items():
primitive[key] = self._element_type.to_primitive(
obj, '%s["%s"]' % (attr, key), element)
return primitive
def from_primitive(self, obj, attr, value):
concrete = {}
for key, element in value.items():
concrete[key] = self._element_type.from_primitive(
obj, '%s["%s"]' % (attr, key), element)
return concrete
def stringify(self, value):
return '{%s}' % (
','.join(['%s=%s' % (key, self._element_type.stringify(val))
for key, val in sorted(value.items())]))
class DictProxyField(object):
"""Descriptor allowing us to assign pinning data as a dict of key_types.
This allows us to have an object field that will be a dict of key_type
keys, allowing that will convert back to string-keyed dict.
This will take care of the conversion while the dict field will make sure
that we store the raw json-serializable data on the object.
key_type should return a type that unambiguously responds to six.text_type
so that calling key_type on it yields the same thing.
"""
def __init__(self, dict_field_name, key_type=int):
self._fld_name = dict_field_name
self._key_type = key_type
def __get__(self, obj, obj_type=None):
if obj is None:
return self
if getattr(obj, self._fld_name) is None:
return
return {self._key_type(k): v
for k, v in six.iteritems(getattr(obj, self._fld_name))}
def __set__(self, obj, val):
if val is None:
setattr(obj, self._fld_name, val)
else:
setattr(obj, self._fld_name, {six.text_type(k): v
for k, v in six.iteritems(val)})
class Set(CompoundFieldType):
def coerce(self, obj, attr, value):
if not isinstance(value, set):
raise ValueError(_('A set is required here'))
coerced = set()
for element in value:
coerced.add(self._element_type.coerce(
obj, '%s["%s"]' % (attr, element), element))
return coerced
def to_primitive(self, obj, attr, value):
return tuple(
self._element_type.to_primitive(obj, attr, x) for x in value)
def from_primitive(self, obj, attr, value):
return set([self._element_type.from_primitive(obj, attr, x)
for x in value])
def stringify(self, value):
return 'set([%s])' % (
','.join([self._element_type.stringify(x) for x in value]))
class Object(FieldType):
def __init__(self, obj_name, **kwargs):
self._obj_name = obj_name
super(Object, self).__init__(**kwargs)
def coerce(self, obj, attr, value):
try:
obj_name = value.obj_name()
except AttributeError:
obj_name = ""
if obj_name != self._obj_name:
raise ValueError(_('An object of type %s is required here') %
self._obj_name)
return value
@staticmethod
def to_primitive(obj, attr, value):
return value.obj_to_primitive()
@staticmethod
def from_primitive(obj, attr, value):
# FIXME(danms): Avoid circular import from base.py
from cinder.objects import base as obj_base
# NOTE (ndipanov): If they already got hydrated by the serializer, just
# pass them back unchanged
if isinstance(value, obj_base.CinderObject):
return value
return obj_base.CinderObject.obj_from_primitive(value, obj._context)
def describe(self):
return "Object<%s>" % self._obj_name
def stringify(self, value):
if 'uuid' in value.fields:
ident = '(%s)' % (value.obj_attr_is_set('uuid') and value.uuid or
'UNKNOWN')
elif 'id' in value.fields:
ident = '(%s)' % (value.obj_attr_is_set('id') and value.id or
'UNKNOWN')
else:
ident = ''
return '%s%s' % (self._obj_name, ident)
class AutoTypedField(Field):
AUTO_TYPE = None
def __init__(self, **kwargs):
super(AutoTypedField, self).__init__(self.AUTO_TYPE, **kwargs)
class StringField(AutoTypedField):
AUTO_TYPE = String()
class UUIDField(AutoTypedField):
AUTO_TYPE = UUID()
class IntegerField(AutoTypedField):
AUTO_TYPE = Integer()
class FloatField(AutoTypedField):
AUTO_TYPE = Float()
class BooleanField(AutoTypedField):
AUTO_TYPE = Boolean()
class DateTimeField(AutoTypedField):
AUTO_TYPE = DateTime()
class DictOfStringsField(AutoTypedField):
AUTO_TYPE = Dict(String())
class DictOfNullableStringsField(AutoTypedField):
AUTO_TYPE = Dict(String(), nullable=True)
class DictOfIntegersField(AutoTypedField):
AUTO_TYPE = Dict(Integer())
class ListOfStringsField(AutoTypedField):
AUTO_TYPE = List(String())
class SetOfIntegersField(AutoTypedField):
AUTO_TYPE = Set(Integer())
class ListOfSetsOfIntegersField(AutoTypedField):
AUTO_TYPE = List(Set(Integer()))
class ListOfDictOfNullableStringsField(AutoTypedField):
AUTO_TYPE = List(Dict(String(), nullable=True))
class ObjectField(AutoTypedField):
def __init__(self, objtype, **kwargs):
self.AUTO_TYPE = Object(objtype)
super(ObjectField, self).__init__(**kwargs)
class ListOfObjectsField(AutoTypedField):
def __init__(self, objtype, **kwargs):
self.AUTO_TYPE = List(Object(objtype))
super(ListOfObjectsField, self).__init__(**kwargs)

View File

@ -20,6 +20,7 @@ from oslo import messaging
from oslo_config import cfg
from oslo_serialization import jsonutils
from cinder.objects import base as objects_base
from cinder import rpc
@ -48,7 +49,9 @@ class SchedulerAPI(object):
super(SchedulerAPI, self).__init__()
target = messaging.Target(topic=CONF.scheduler_topic,
version=self.RPC_API_VERSION)
self.client = rpc.get_client(target, version_cap='1.7')
serializer = objects_base.CinderObjectSerializer()
self.client = rpc.get_client(target, version_cap='1.7',
serializer=serializer)
def create_consistencygroup(self, ctxt, topic, group_id,
request_spec_list=None,

View File

@ -35,6 +35,7 @@ from cinder import context
from cinder import db
from cinder import exception
from cinder.i18n import _
from cinder.objects import base as objects_base
from cinder.openstack.common import log as logging
from cinder.openstack.common import loopingcall
from cinder.openstack.common import service
@ -154,7 +155,8 @@ class Service(service.Service):
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
endpoints.extend(self.manager.additional_endpoints)
self.rpcserver = rpc.get_server(target, endpoints)
serializer = objects_base.CinderObjectSerializer()
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
if self.report_interval:

View File

View File

@ -0,0 +1,316 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import iso8601
from oslo_utils import timeutils
from cinder.objects import base as obj_base
from cinder.objects import fields
from cinder import test
class FakeFieldType(fields.FieldType):
def coerce(self, obj, attr, value):
return '*%s*' % value
def to_primitive(self, obj, attr, value):
return '!%s!' % value
def from_primitive(self, obj, attr, value):
return value[1:-1]
class TestField(test.TestCase):
def setUp(self):
super(TestField, self).setUp()
self.field = fields.Field(FakeFieldType())
self.coerce_good_values = [('foo', '*foo*')]
self.coerce_bad_values = []
self.to_primitive_values = [('foo', '!foo!')]
self.from_primitive_values = [('!foo!', 'foo')]
def test_coerce_good_values(self):
for in_val, out_val in self.coerce_good_values:
self.assertEqual(out_val, self.field.coerce('obj', 'attr', in_val))
def test_coerce_bad_values(self):
for in_val in self.coerce_bad_values:
self.assertRaises((TypeError, ValueError),
self.field.coerce, 'obj', 'attr', in_val)
def test_to_primitive(self):
for in_val, prim_val in self.to_primitive_values:
self.assertEqual(prim_val, self.field.to_primitive('obj', 'attr',
in_val))
def test_from_primitive(self):
class ObjectLikeThing(object):
_context = 'context'
for prim_val, out_val in self.from_primitive_values:
self.assertEqual(out_val, self.field.from_primitive(
ObjectLikeThing, 'attr', prim_val))
def test_stringify(self):
self.assertEqual('123', self.field.stringify(123))
class TestString(TestField):
def setUp(self):
super(TestField, self).setUp()
self.field = fields.StringField()
self.coerce_good_values = [('foo', 'foo'), (1, '1'), (1L, '1'),
(True, 'True')]
self.coerce_bad_values = [None]
self.to_primitive_values = self.coerce_good_values[0:1]
self.from_primitive_values = self.coerce_good_values[0:1]
def test_stringify(self):
self.assertEqual("'123'", self.field.stringify(123))
class TestInteger(TestField):
def setUp(self):
super(TestField, self).setUp()
self.field = fields.IntegerField()
self.coerce_good_values = [(1, 1), ('1', 1)]
self.coerce_bad_values = ['foo', None]
self.to_primitive_values = self.coerce_good_values[0:1]
self.from_primitive_values = self.coerce_good_values[0:1]
class TestFloat(TestField):
def setUp(self):
super(TestFloat, self).setUp()
self.field = fields.FloatField()
self.coerce_good_values = [(1.1, 1.1), ('1.1', 1.1)]
self.coerce_bad_values = ['foo', None]
self.to_primitive_values = self.coerce_good_values[0:1]
self.from_primitive_values = self.coerce_good_values[0:1]
class TestBoolean(TestField):
def setUp(self):
super(TestField, self).setUp()
self.field = fields.BooleanField()
self.coerce_good_values = [(True, True), (False, False), (1, True),
('foo', True), (0, False), ('', False)]
self.coerce_bad_values = []
self.to_primitive_values = self.coerce_good_values[0:2]
self.from_primitive_values = self.coerce_good_values[0:2]
class TestDateTime(TestField):
def setUp(self):
super(TestDateTime, self).setUp()
self.dt = datetime.datetime(1955, 11, 5, tzinfo=iso8601.iso8601.Utc())
self.field = fields.DateTimeField()
self.coerce_good_values = [(self.dt, self.dt),
(timeutils.isotime(self.dt), self.dt)]
self.coerce_bad_values = [1, 'foo']
self.to_primitive_values = [(self.dt, timeutils.isotime(self.dt))]
self.from_primitive_values = [(timeutils.isotime(self.dt), self.dt)]
def test_stringify(self):
self.assertEqual(
'1955-11-05T18:00:00Z',
self.field.stringify(
datetime.datetime(1955, 11, 5, 18, 0, 0,
tzinfo=iso8601.iso8601.Utc())))
class TestDict(TestField):
def setUp(self):
super(TestDict, self).setUp()
self.field = fields.Field(fields.Dict(FakeFieldType()))
self.coerce_good_values = [({'foo': 'bar'}, {'foo': '*bar*'}),
({'foo': 1}, {'foo': '*1*'})]
self.coerce_bad_values = [{1: 'bar'}, 'foo']
self.to_primitive_values = [({'foo': 'bar'}, {'foo': '!bar!'})]
self.from_primitive_values = [({'foo': '!bar!'}, {'foo': 'bar'})]
def test_stringify(self):
self.assertEqual("{key=val}", self.field.stringify({'key': 'val'}))
class TestDictOfStrings(TestField):
def setUp(self):
super(TestDictOfStrings, self).setUp()
self.field = fields.DictOfStringsField()
self.coerce_good_values = [({'foo': 'bar'}, {'foo': 'bar'}),
({'foo': 1}, {'foo': '1'})]
self.coerce_bad_values = [{1: 'bar'}, {'foo': None}, 'foo']
self.to_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})]
self.from_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})]
def test_stringify(self):
self.assertEqual("{key='val'}", self.field.stringify({'key': 'val'}))
class TestDictOfIntegers(TestField):
def setUp(self):
super(TestDictOfIntegers, self).setUp()
self.field = fields.DictOfIntegersField()
self.coerce_good_values = [({'foo': '42'}, {'foo': 42}),
({'foo': 4.2}, {'foo': 4})]
self.coerce_bad_values = [{1: 'bar'}, {'foo': 'boo'},
'foo', {'foo': None}]
self.to_primitive_values = [({'foo': 42}, {'foo': 42})]
self.from_primitive_values = [({'foo': 42}, {'foo': 42})]
def test_stringify(self):
self.assertEqual("{key=42}", self.field.stringify({'key': 42}))
class TestDictOfStringsNone(TestField):
def setUp(self):
super(TestDictOfStringsNone, self).setUp()
self.field = fields.DictOfNullableStringsField()
self.coerce_good_values = [({'foo': 'bar'}, {'foo': 'bar'}),
({'foo': 1}, {'foo': '1'}),
({'foo': None}, {'foo': None})]
self.coerce_bad_values = [{1: 'bar'}, 'foo']
self.to_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})]
self.from_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})]
def test_stringify(self):
self.assertEqual("{k2=None,key='val'}",
self.field.stringify({'k2': None,
'key': 'val'}))
class TestListOfDictOfNullableStringsField(TestField):
def setUp(self):
super(TestListOfDictOfNullableStringsField, self).setUp()
self.field = fields.ListOfDictOfNullableStringsField()
self.coerce_good_values = [([{'f': 'b', 'f1': 'b1'}, {'f2': 'b2'}],
[{'f': 'b', 'f1': 'b1'}, {'f2': 'b2'}]),
([{'f': 1}, {'f1': 'b1'}],
[{'f': '1'}, {'f1': 'b1'}]),
([{'foo': None}], [{'foo': None}])]
self.coerce_bad_values = [[{1: 'a'}], ['ham', 1], ['eggs']]
self.to_primitive_values = [([{'f': 'b'}, {'f1': 'b1'}, {'f2': None}],
[{'f': 'b'}, {'f1': 'b1'}, {'f2': None}])]
self.from_primitive_values = [([{'f': 'b'}, {'f1': 'b1'},
{'f2': None}],
[{'f': 'b'}, {'f1': 'b1'},
{'f2': None}])]
def test_stringify(self):
self.assertEqual("[{f=None,f1='b1'},{f2='b2'}]",
self.field.stringify(
[{'f': None, 'f1': 'b1'}, {'f2': 'b2'}]))
class TestList(TestField):
def setUp(self):
super(TestList, self).setUp()
self.field = fields.Field(fields.List(FakeFieldType()))
self.coerce_good_values = [(['foo', 'bar'], ['*foo*', '*bar*'])]
self.coerce_bad_values = ['foo']
self.to_primitive_values = [(['foo'], ['!foo!'])]
self.from_primitive_values = [(['!foo!'], ['foo'])]
def test_stringify(self):
self.assertEqual('[123]', self.field.stringify([123]))
class TestListOfStrings(TestField):
def setUp(self):
super(TestListOfStrings, self).setUp()
self.field = fields.ListOfStringsField()
self.coerce_good_values = [(['foo', 'bar'], ['foo', 'bar'])]
self.coerce_bad_values = ['foo']
self.to_primitive_values = [(['foo'], ['foo'])]
self.from_primitive_values = [(['foo'], ['foo'])]
def test_stringify(self):
self.assertEqual("['abc']", self.field.stringify(['abc']))
class TestSet(TestField):
def setUp(self):
super(TestSet, self).setUp()
self.field = fields.Field(fields.Set(FakeFieldType()))
self.coerce_good_values = [(set(['foo', 'bar']),
set(['*foo*', '*bar*']))]
self.coerce_bad_values = [['foo'], {'foo': 'bar'}]
self.to_primitive_values = [(set(['foo']), tuple(['!foo!']))]
self.from_primitive_values = [(tuple(['!foo!']), set(['foo']))]
def test_stringify(self):
self.assertEqual('set([123])', self.field.stringify(set([123])))
class TestSetOfIntegers(TestField):
def setUp(self):
super(TestSetOfIntegers, self).setUp()
self.field = fields.SetOfIntegersField()
self.coerce_good_values = [(set(['1', 2]),
set([1, 2]))]
self.coerce_bad_values = [set(['foo'])]
self.to_primitive_values = [(set([1]), tuple([1]))]
self.from_primitive_values = [(tuple([1]), set([1]))]
def test_stringify(self):
self.assertEqual('set([1,2])', self.field.stringify(set([1, 2])))
class TestListOfSetsOfIntegers(TestField):
def setUp(self):
super(TestListOfSetsOfIntegers, self).setUp()
self.field = fields.ListOfSetsOfIntegersField()
self.coerce_good_values = [([set(['1', 2]), set([3, '4'])],
[set([1, 2]), set([3, 4])])]
self.coerce_bad_values = [[set(['foo'])]]
self.to_primitive_values = [([set([1])], [tuple([1])])]
self.from_primitive_values = [([tuple([1])], [set([1])])]
def test_stringify(self):
self.assertEqual('[set([1,2])]', self.field.stringify([set([1, 2])]))
class TestObject(TestField):
def setUp(self):
super(TestObject, self).setUp()
class TestableObject(obj_base.CinderObject):
fields = {
'uuid': fields.StringField(),
}
def __eq__(self, value):
# NOTE(danms): Be rather lax about this equality thing to
# satisfy the assertEqual() in test_from_primitive(). We
# just want to make sure the right type of object is re-created
return value.__class__.__name__ == TestableObject.__name__
class OtherTestableObject(obj_base.CinderObject):
pass
test_inst = TestableObject()
self._test_cls = TestableObject
self.field = fields.Field(fields.Object('TestableObject'))
self.coerce_good_values = [(test_inst, test_inst)]
self.coerce_bad_values = [OtherTestableObject(), 1, 'foo']
self.to_primitive_values = [(test_inst, test_inst.obj_to_primitive())]
self.from_primitive_values = [(test_inst.obj_to_primitive(),
test_inst), (test_inst, test_inst)]
def test_stringify(self):
obj = self._test_cls(uuid='fake-uuid')
self.assertEqual('TestableObject(fake-uuid)',
self.field.stringify(obj))

View File

@ -0,0 +1,968 @@
# Copyright 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import copy
import datetime
import mock
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import six
from testtools import matchers
from cinder import context
from cinder import exception
from cinder import objects
from cinder.objects import base
from cinder.objects import fields
from cinder import test
from cinder.tests import fake_notifier
class MyOwnedObject(base.CinderPersistentObject, base.CinderObject):
VERSION = '1.0'
fields = {'baz': fields.Field(fields.Integer())}
class MyObj(base.CinderPersistentObject, base.CinderObject,
base.CinderObjectDictCompat):
VERSION = '1.6'
fields = {'foo': fields.Field(fields.Integer(), default=1),
'bar': fields.Field(fields.String()),
'missing': fields.Field(fields.String()),
'readonly': fields.Field(fields.Integer(), read_only=True),
'rel_object': fields.ObjectField('MyOwnedObject', nullable=True),
'rel_objects': fields.ListOfObjectsField('MyOwnedObject',
nullable=True),
}
@staticmethod
def _from_db_object(context, obj, db_obj):
self = MyObj()
self.foo = db_obj['foo']
self.bar = db_obj['bar']
self.missing = db_obj['missing']
self.readonly = 1
return self
def obj_load_attr(self, attrname):
setattr(self, attrname, 'loaded!')
@base.remotable_classmethod
def query(cls, context):
obj = cls(context=context, foo=1, bar='bar')
obj.obj_reset_changes()
return obj
@base.remotable
def marco(self, context):
return 'polo'
@base.remotable
def _update_test(self, context):
if context.project_id == 'alternate':
self.bar = 'alternate-context'
else:
self.bar = 'updated'
@base.remotable
def save(self, context):
self.obj_reset_changes()
@base.remotable
def refresh(self, context):
self.foo = 321
self.bar = 'refreshed'
self.obj_reset_changes()
@base.remotable
def modify_save_modify(self, context):
self.bar = 'meow'
self.save()
self.foo = 42
self.rel_object = MyOwnedObject(baz=42)
def obj_make_compatible(self, primitive, target_version):
super(MyObj, self).obj_make_compatible(primitive, target_version)
# NOTE(danms): Simulate an older version that had a different
# format for the 'bar' attribute
if target_version == '1.1' and 'bar' in primitive:
primitive['bar'] = 'old%s' % primitive['bar']
class MyObjDiffVers(MyObj):
VERSION = '1.5'
@classmethod
def obj_name(cls):
return 'MyObj'
class MyObj2(object):
@classmethod
def obj_name(cls):
return 'MyObj'
@base.remotable_classmethod
def query(cls, *args, **kwargs):
pass
class RandomMixInWithNoFields(object):
"""Used to test object inheritance using a mixin that has no fields."""
pass
class TestSubclassedObject(RandomMixInWithNoFields, MyObj):
fields = {'new_field': fields.Field(fields.String())}
class TestMetaclass(test.TestCase):
def test_obj_tracking(self):
@six.add_metaclass(base.CinderObjectMetaclass)
class NewBaseClass(object):
VERSION = '1.0'
fields = {}
@classmethod
def obj_name(cls):
return cls.__name__
class Fake1TestObj1(NewBaseClass):
@classmethod
def obj_name(cls):
return 'fake1'
class Fake1TestObj2(Fake1TestObj1):
pass
class Fake1TestObj3(Fake1TestObj1):
VERSION = '1.1'
class Fake2TestObj1(NewBaseClass):
@classmethod
def obj_name(cls):
return 'fake2'
class Fake1TestObj4(Fake1TestObj3):
VERSION = '1.2'
class Fake2TestObj2(Fake2TestObj1):
VERSION = '1.1'
class Fake1TestObj5(Fake1TestObj1):
VERSION = '1.1'
# Newest versions first in the list. Duplicate versions take the
# newest object.
expected = {'fake1': [Fake1TestObj4, Fake1TestObj5, Fake1TestObj2],
'fake2': [Fake2TestObj2, Fake2TestObj1]}
self.assertEqual(expected, NewBaseClass._obj_classes)
# The following should work, also.
self.assertEqual(expected, Fake1TestObj1._obj_classes)
self.assertEqual(expected, Fake1TestObj2._obj_classes)
self.assertEqual(expected, Fake1TestObj3._obj_classes)
self.assertEqual(expected, Fake1TestObj4._obj_classes)
self.assertEqual(expected, Fake1TestObj5._obj_classes)
self.assertEqual(expected, Fake2TestObj1._obj_classes)
self.assertEqual(expected, Fake2TestObj2._obj_classes)
def test_field_checking(self):
def create_class(field):
class TestField(base.CinderObject):
VERSION = '1.5'
fields = {'foo': field()}
return TestField
create_class(fields.BooleanField)
self.assertRaises(exception.ObjectFieldInvalid,
create_class, fields.Boolean)
self.assertRaises(exception.ObjectFieldInvalid,
create_class, int)
class TestObjToPrimitive(test.TestCase):
def test_obj_to_primitive_list(self):
class MyObjElement(base.CinderObject):
fields = {'foo': fields.IntegerField()}
def __init__(self, foo):
super(MyObjElement, self).__init__()
self.foo = foo
class MyList(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('MyObjElement')}
mylist = MyList()
mylist.objects = [MyObjElement(1), MyObjElement(2), MyObjElement(3)]
self.assertEqual([1, 2, 3],
[x['foo'] for x in base.obj_to_primitive(mylist)])
def test_obj_to_primitive_dict(self):
myobj = MyObj(foo=1, bar='foo')
self.assertEqual({'foo': 1, 'bar': 'foo'},
base.obj_to_primitive(myobj))
def test_obj_to_primitive_recursive(self):
class MyList(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('MyObj')}
mylist = MyList(objects=[MyObj(), MyObj()])
for i, value in enumerate(mylist):
value.foo = i
self.assertEqual([{'foo': 0}, {'foo': 1}],
base.obj_to_primitive(mylist))
class TestObjMakeList(test.TestCase):
def test_obj_make_list(self):
class MyList(base.ObjectListBase, base.CinderObject):
pass
db_objs = [{'foo': 1, 'bar': 'baz', 'missing': 'banana'},
{'foo': 2, 'bar': 'bat', 'missing': 'apple'},
]
mylist = base.obj_make_list('ctxt', MyList(), MyObj, db_objs)
self.assertEqual(2, len(mylist))
self.assertEqual('ctxt', mylist._context)
for index, item in enumerate(mylist):
self.assertEqual(db_objs[index]['foo'], item.foo)
self.assertEqual(db_objs[index]['bar'], item.bar)
self.assertEqual(db_objs[index]['missing'], item.missing)
def compare_obj(test, obj, db_obj, subs=None, allow_missing=None,
comparators=None):
"""Compare a CinderObject and a dict-like database object.
This automatically converts TZ-aware datetimes and iterates over
the fields of the object.
:param:test: The TestCase doing the comparison
:param:obj: The CinderObject to examine
:param:db_obj: The dict-like database object to use as reference
:param:subs: A dict of objkey=dbkey field substitutions
:param:allow_missing: A list of fields that may not be in db_obj
:param:comparators: Map of comparator functions to use for certain fields
"""
if subs is None:
subs = {}
if allow_missing is None:
allow_missing = []
if comparators is None:
comparators = {}
for key in obj.fields:
if key in allow_missing and not obj.obj_attr_is_set(key):
continue
obj_val = getattr(obj, key)
db_key = subs.get(key, key)
db_val = db_obj[db_key]
if isinstance(obj_val, datetime.datetime):
obj_val = obj_val.replace(tzinfo=None)
if key in comparators:
comparator = comparators[key]
comparator(db_val, obj_val)
else:
test.assertEqual(db_val, obj_val)
class _BaseTestCase(test.TestCase):
def setUp(self):
super(_BaseTestCase, self).setUp()
self.remote_object_calls = list()
self.user_id = 'fake-user'
self.project_id = 'fake-project'
self.context = context.RequestContext(self.user_id, self.project_id)
fake_notifier.stub_notifier(self.stubs)
self.addCleanup(fake_notifier.reset)
def compare_obj(self, obj, db_obj, subs=None, allow_missing=None,
comparators=None):
compare_obj(self, obj, db_obj, subs=subs, allow_missing=allow_missing,
comparators=comparators)
def json_comparator(self, expected, obj_val):
# json-ify an object field for comparison with its db str
# equivalent
self.assertEqual(expected, jsonutils.dumps(obj_val))
def str_comparator(self, expected, obj_val):
"""Compare an object field to a string in the db by performing
a simple coercion on the object field value.
"""
self.assertEqual(expected, str(obj_val))
def assertNotIsInstance(self, obj, cls, msg=None):
"""Python < v2.7 compatibility. Assert 'not isinstance(obj, cls)."""
try:
f = super(_BaseTestCase, self).assertNotIsInstance
except AttributeError:
self.assertThat(obj,
matchers.Not(matchers.IsInstance(cls)),
message=msg or '')
else:
f(obj, cls, msg=msg)
class _LocalTest(_BaseTestCase):
def setUp(self):
super(_LocalTest, self).setUp()
# Just in case
base.CinderObject.indirection_api = None
def assertRemotes(self):
self.assertEqual(self.remote_object_calls, [])
@contextlib.contextmanager
def things_temporarily_local():
_api = base.CinderObject.indirection_api
base.CinderObject.indirection_api = None
yield
base.CinderObject.indirection_api = _api
class _TestObject(object):
def test_object_attrs_in_init(self):
# Now check the test one in this file. Should be newest version
self.assertEqual('1.6', objects.MyObj.VERSION)
def test_hydration_type_error(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.5',
'cinder_object.data': {'foo': 'a'}}
self.assertRaises(ValueError, MyObj.obj_from_primitive, primitive)
def test_hydration(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.5',
'cinder_object.data': {'foo': 1}}
real_method = MyObj._obj_from_primitive
def _obj_from_primitive(*args):
return real_method(*args)
with mock.patch.object(MyObj, '_obj_from_primitive') as ofp:
ofp.side_effect = _obj_from_primitive
obj = MyObj.obj_from_primitive(primitive)
ofp.assert_called_once_with(None, '1.5', primitive)
self.assertEqual(obj.foo, 1)
def test_hydration_version_different(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.2',
'cinder_object.data': {'foo': 1}}
obj = MyObj.obj_from_primitive(primitive)
self.assertEqual(obj.foo, 1)
self.assertEqual('1.2', obj.VERSION)
def test_hydration_bad_ns(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'foo',
'cinder_object.version': '1.5',
'cinder_object.data': {'foo': 1}}
self.assertRaises(exception.UnsupportedObjectError,
MyObj.obj_from_primitive, primitive)
def test_hydration_additional_unexpected_stuff(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.5.1',
'cinder_object.data': {
'foo': 1,
'unexpected_thing': 'foobar'}}
obj = MyObj.obj_from_primitive(primitive)
self.assertEqual(1, obj.foo)
self.assertFalse(hasattr(obj, 'unexpected_thing'))
# NOTE(danms): If we call obj_from_primitive() directly
# with a version containing .z, we'll get that version
# in the resulting object. In reality, when using the
# serializer, we'll get that snipped off (tested
# elsewhere)
self.assertEqual('1.5.1', obj.VERSION)
def test_dehydration(self):
expected = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.6',
'cinder_object.data': {'foo': 1}}
obj = MyObj(foo=1)
obj.obj_reset_changes()
self.assertEqual(obj.obj_to_primitive(), expected)
def test_object_property(self):
obj = MyObj(foo=1)
self.assertEqual(obj.foo, 1)
def test_object_property_type_error(self):
obj = MyObj()
def fail():
obj.foo = 'a'
self.assertRaises(ValueError, fail)
def test_object_dict_syntax(self):
obj = MyObj(foo=123, bar='bar')
self.assertEqual(obj['foo'], 123)
self.assertEqual(sorted(obj.items(), key=lambda x: x[0]),
[('bar', 'bar'), ('foo', 123)])
self.assertEqual(sorted(list(obj.iteritems()), key=lambda x: x[0]),
[('bar', 'bar'), ('foo', 123)])
def test_load(self):
obj = MyObj()
self.assertEqual(obj.bar, 'loaded!')
def test_load_in_base(self):
class Foo(base.CinderObject):
fields = {'foobar': fields.Field(fields.Integer())}
obj = Foo()
with self.assertRaisesRegex(NotImplementedError, ".*foobar.*"):
obj.foobar
def test_loaded_in_primitive(self):
obj = MyObj(foo=1)
obj.obj_reset_changes()
self.assertEqual(obj.bar, 'loaded!')
expected = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.6',
'cinder_object.changes': ['bar'],
'cinder_object.data': {'foo': 1,
'bar': 'loaded!'}}
self.assertEqual(obj.obj_to_primitive(), expected)
def test_changes_in_primitive(self):
obj = MyObj(foo=123)
self.assertEqual(obj.obj_what_changed(), set(['foo']))
primitive = obj.obj_to_primitive()
self.assertIn('cinder_object.changes', primitive)
obj2 = MyObj.obj_from_primitive(primitive)
self.assertEqual(obj2.obj_what_changed(), set(['foo']))
obj2.obj_reset_changes()
self.assertEqual(obj2.obj_what_changed(), set())
def test_obj_class_from_name(self):
obj = base.CinderObject.obj_class_from_name('MyObj', '1.5')
self.assertEqual('1.5', obj.VERSION)
def test_obj_class_from_name_latest_compatible(self):
obj = base.CinderObject.obj_class_from_name('MyObj', '1.1')
self.assertEqual('1.6', obj.VERSION)
def test_unknown_objtype(self):
self.assertRaises(exception.UnsupportedObjectError,
base.CinderObject.obj_class_from_name, 'foo', '1.0')
def test_obj_class_from_name_supported_version(self):
error = None
try:
base.CinderObject.obj_class_from_name('MyObj', '1.25')
except exception.IncompatibleObjectVersion as error:
pass
self.assertIsNotNone(error)
self.assertEqual('1.6', error.kwargs['supported'])
def test_with_alternate_context(self):
ctxt1 = context.RequestContext('foo', 'foo')
ctxt2 = context.RequestContext('bar', 'alternate')
obj = MyObj.query(ctxt1)
obj._update_test(ctxt2)
self.assertEqual(obj.bar, 'alternate-context')
self.assertRemotes()
def test_orphaned_object(self):
obj = MyObj.query(self.context)
obj._context = None
self.assertRaises(exception.OrphanedObjectError,
obj._update_test)
self.assertRemotes()
def test_changed_1(self):
obj = MyObj.query(self.context)
obj.foo = 123
self.assertEqual(obj.obj_what_changed(), set(['foo']))
obj._update_test(self.context)
self.assertEqual(obj.obj_what_changed(), set(['foo', 'bar']))
self.assertEqual(obj.foo, 123)
self.assertRemotes()
def test_changed_2(self):
obj = MyObj.query(self.context)
obj.foo = 123
self.assertEqual(obj.obj_what_changed(), set(['foo']))
obj.save()
self.assertEqual(obj.obj_what_changed(), set([]))
self.assertEqual(obj.foo, 123)
self.assertRemotes()
def test_changed_3(self):
obj = MyObj.query(self.context)
obj.foo = 123
self.assertEqual(obj.obj_what_changed(), set(['foo']))
obj.refresh()
self.assertEqual(obj.obj_what_changed(), set([]))
self.assertEqual(obj.foo, 321)
self.assertEqual(obj.bar, 'refreshed')
self.assertRemotes()
def test_changed_4(self):
obj = MyObj.query(self.context)
obj.bar = 'something'
self.assertEqual(obj.obj_what_changed(), set(['bar']))
obj.modify_save_modify(self.context)
self.assertEqual(obj.obj_what_changed(), set(['foo', 'rel_object']))
self.assertEqual(obj.foo, 42)
self.assertEqual(obj.bar, 'meow')
self.assertIsInstance(obj.rel_object, MyOwnedObject)
self.assertRemotes()
def test_changed_with_sub_object(self):
class ParentObject(base.CinderObject):
fields = {'foo': fields.IntegerField(),
'bar': fields.ObjectField('MyObj'),
}
obj = ParentObject()
self.assertEqual(set(), obj.obj_what_changed())
obj.foo = 1
self.assertEqual(set(['foo']), obj.obj_what_changed())
bar = MyObj()
obj.bar = bar
self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed())
obj.obj_reset_changes()
self.assertEqual(set(), obj.obj_what_changed())
bar.foo = 1
self.assertEqual(set(['bar']), obj.obj_what_changed())
def test_static_result(self):
obj = MyObj.query(self.context)
self.assertEqual(obj.bar, 'bar')
result = obj.marco()
self.assertEqual(result, 'polo')
self.assertRemotes()
def test_updates(self):
obj = MyObj.query(self.context)
self.assertEqual(obj.foo, 1)
obj._update_test()
self.assertEqual(obj.bar, 'updated')
self.assertRemotes()
def test_base_attributes(self):
dt = datetime.datetime(1955, 11, 5)
obj = MyObj(created_at=dt, updated_at=dt, deleted_at=None,
deleted=False)
expected = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.6',
'cinder_object.changes':
['deleted', 'created_at', 'deleted_at', 'updated_at'],
'cinder_object.data':
{'created_at': timeutils.isotime(dt),
'updated_at': timeutils.isotime(dt),
'deleted_at': None,
'deleted': False,
}
}
self.assertEqual(obj.obj_to_primitive(), expected)
def test_contains(self):
obj = MyObj()
self.assertNotIn('foo', obj)
obj.foo = 1
self.assertIn('foo', obj)
self.assertNotIn('does_not_exist', obj)
def test_obj_attr_is_set(self):
obj = MyObj(foo=1)
self.assertTrue(obj.obj_attr_is_set('foo'))
self.assertFalse(obj.obj_attr_is_set('bar'))
self.assertRaises(AttributeError, obj.obj_attr_is_set, 'bang')
def test_get(self):
obj = MyObj(foo=1)
# Foo has value, should not get the default
self.assertEqual(1, obj.get('foo', 2))
# Foo has value, should return the value without error
self.assertEqual(1, obj.get('foo'))
# Bar is not loaded, so we should get the default
self.assertEqual('not-loaded', obj.get('bar', 'not-loaded'))
# Bar without a default should lazy-load
self.assertEqual('loaded!', obj.get('bar'))
# Bar now has a default, but loaded value should be returned
self.assertEqual('loaded!', obj.get('bar', 'not-loaded'))
# Invalid attribute should raise AttributeError
self.assertRaises(AttributeError, obj.get, 'nothing')
# ...even with a default
self.assertRaises(AttributeError, obj.get, 'nothing', 3)
def test_object_inheritance(self):
base_fields = base.CinderPersistentObject.fields.keys()
myobj_fields = (['foo', 'bar', 'missing',
'readonly', 'rel_object', 'rel_objects'] +
base_fields)
myobj3_fields = ['new_field']
self.assertTrue(issubclass(TestSubclassedObject, MyObj))
self.assertEqual(len(myobj_fields), len(MyObj.fields))
self.assertEqual(set(myobj_fields), set(MyObj.fields.keys()))
self.assertEqual(len(myobj_fields) + len(myobj3_fields),
len(TestSubclassedObject.fields))
self.assertEqual(set(myobj_fields) | set(myobj3_fields),
set(TestSubclassedObject.fields.keys()))
def test_obj_as_admin(self):
obj = MyObj(context=self.context)
def fake(*args, **kwargs):
self.assertTrue(obj._context.is_admin)
with mock.patch.object(obj, 'obj_reset_changes') as mock_fn:
mock_fn.side_effect = fake
with obj.obj_as_admin():
obj.save()
self.assertTrue(mock_fn.called)
self.assertFalse(obj._context.is_admin)
def test_obj_as_admin_orphaned(self):
def testme():
obj = MyObj()
with obj.obj_as_admin():
pass
self.assertRaises(exception.OrphanedObjectError, testme)
def test_get_changes(self):
obj = MyObj()
self.assertEqual({}, obj.obj_get_changes())
obj.foo = 123
self.assertEqual({'foo': 123}, obj.obj_get_changes())
obj.bar = 'test'
self.assertEqual({'foo': 123, 'bar': 'test'}, obj.obj_get_changes())
obj.obj_reset_changes()
self.assertEqual({}, obj.obj_get_changes())
def test_obj_fields(self):
class TestObj(base.CinderObject):
fields = {'foo': fields.Field(fields.Integer())}
obj_extra_fields = ['bar']
@property
def bar(self):
return 'this is bar'
obj = TestObj()
self.assertEqual(['foo', 'bar'], obj.obj_fields)
def test_obj_constructor(self):
obj = MyObj(context=self.context, foo=123, bar='abc')
self.assertEqual(123, obj.foo)
self.assertEqual('abc', obj.bar)
self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed())
def test_obj_read_only(self):
obj = MyObj(context=self.context, foo=123, bar='abc')
obj.readonly = 1
self.assertRaises(exception.ReadOnlyFieldError, setattr,
obj, 'readonly', 2)
def test_obj_repr(self):
obj = MyObj(foo=123)
self.assertEqual('MyObj(bar=<?>,created_at=<?>,deleted=<?>,'
'deleted_at=<?>,foo=123,missing=<?>,readonly=<?>,'
'rel_object=<?>,rel_objects=<?>,updated_at=<?>)',
repr(obj))
def test_obj_make_obj_compatible(self):
subobj = MyOwnedObject(baz=1)
obj = MyObj(rel_object=subobj)
obj.obj_relationships = {
'rel_object': [('1.5', '1.1'), ('1.7', '1.2')],
}
primitive = obj.obj_to_primitive()['cinder_object.data']
with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
obj._obj_make_obj_compatible(copy.copy(primitive), '1.8',
'rel_object')
self.assertFalse(mock_compat.called)
with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
obj._obj_make_obj_compatible(copy.copy(primitive),
'1.7', 'rel_object')
mock_compat.assert_called_once_with(
primitive['rel_object']['cinder_object.data'], '1.2')
self.assertEqual('1.2',
primitive['rel_object']['cinder_object.version'])
with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
obj._obj_make_obj_compatible(copy.copy(primitive),
'1.6', 'rel_object')
mock_compat.assert_called_once_with(
primitive['rel_object']['cinder_object.data'], '1.1')
self.assertEqual('1.1',
primitive['rel_object']['cinder_object.version'])
with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
obj._obj_make_obj_compatible(copy.copy(primitive), '1.5',
'rel_object')
mock_compat.assert_called_once_with(
primitive['rel_object']['cinder_object.data'], '1.1')
self.assertEqual('1.1',
primitive['rel_object']['cinder_object.version'])
with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat:
_prim = copy.copy(primitive)
obj._obj_make_obj_compatible(_prim, '1.4', 'rel_object')
self.assertFalse(mock_compat.called)
self.assertNotIn('rel_object', _prim)
def test_obj_make_compatible_hits_sub_objects(self):
subobj = MyOwnedObject(baz=1)
obj = MyObj(foo=123, rel_object=subobj)
obj.obj_relationships = {'rel_object': [('1.0', '1.0')]}
with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat:
obj.obj_make_compatible({'rel_object': 'foo'}, '1.10')
mock_compat.assert_called_once_with({'rel_object': 'foo'}, '1.10',
'rel_object')
def test_obj_make_compatible_skips_unset_sub_objects(self):
obj = MyObj(foo=123)
obj.obj_relationships = {'rel_object': [('1.0', '1.0')]}
with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat:
obj.obj_make_compatible({'rel_object': 'foo'}, '1.10')
self.assertFalse(mock_compat.called)
def test_obj_make_compatible_complains_about_missing_rules(self):
subobj = MyOwnedObject(baz=1)
obj = MyObj(foo=123, rel_object=subobj)
obj.obj_relationships = {}
self.assertRaises(exception.ObjectActionError,
obj.obj_make_compatible, {}, '1.0')
def test_obj_make_compatible_handles_list_of_objects(self):
subobj = MyOwnedObject(baz=1)
obj = MyObj(rel_objects=[subobj])
obj.obj_relationships = {'rel_objects': [('1.0', '1.123')]}
def fake_make_compat(primitive, version):
self.assertEqual('1.123', version)
self.assertIn('baz', primitive)
with mock.patch.object(subobj, 'obj_make_compatible') as mock_mc:
mock_mc.side_effect = fake_make_compat
obj.obj_to_primitive('1.0')
self.assertTrue(mock_mc.called)
class TestObject(_LocalTest, _TestObject):
def test_set_defaults(self):
obj = MyObj()
obj.obj_set_defaults('foo')
self.assertTrue(obj.obj_attr_is_set('foo'))
self.assertEqual(1, obj.foo)
def test_set_defaults_no_default(self):
obj = MyObj()
self.assertRaises(exception.ObjectActionError,
obj.obj_set_defaults, 'bar')
def test_set_all_defaults(self):
obj = MyObj()
obj.obj_set_defaults()
self.assertEqual(set(['deleted', 'foo']), obj.obj_what_changed())
self.assertEqual(1, obj.foo)
class TestObjectListBase(test.TestCase):
def test_list_like_operations(self):
class MyElement(base.CinderObject):
fields = {'foo': fields.IntegerField()}
def __init__(self, foo):
super(MyElement, self).__init__()
self.foo = foo
class Foo(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('MyElement')}
objlist = Foo(context='foo',
objects=[MyElement(1), MyElement(2), MyElement(3)])
self.assertEqual(list(objlist), objlist.objects)
self.assertEqual(len(objlist), 3)
self.assertIn(objlist.objects[0], objlist)
self.assertEqual(list(objlist[:1]), [objlist.objects[0]])
self.assertEqual(objlist[:1]._context, 'foo')
self.assertEqual(objlist[2], objlist.objects[2])
self.assertEqual(objlist.count(objlist.objects[0]), 1)
self.assertEqual(objlist.index(objlist.objects[1]), 1)
objlist.sort(key=lambda x: x.foo, reverse=True)
self.assertEqual([3, 2, 1],
[x.foo for x in objlist])
def test_serialization(self):
class Foo(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('Bar')}
class Bar(base.CinderObject):
fields = {'foo': fields.Field(fields.String())}
obj = Foo(objects=[])
for i in 'abc':
bar = Bar(foo=i)
obj.objects.append(bar)
obj2 = base.CinderObject.obj_from_primitive(obj.obj_to_primitive())
self.assertFalse(obj is obj2)
self.assertEqual([x.foo for x in obj],
[y.foo for y in obj2])
def test_list_changes(self):
class Foo(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('Bar')}
class Bar(base.CinderObject):
fields = {'foo': fields.StringField()}
obj = Foo(objects=[])
self.assertEqual(set(['objects']), obj.obj_what_changed())
obj.objects.append(Bar(foo='test'))
self.assertEqual(set(['objects']), obj.obj_what_changed())
obj.obj_reset_changes()
# This should still look dirty because the child is dirty
self.assertEqual(set(['objects']), obj.obj_what_changed())
obj.objects[0].obj_reset_changes()
# This should now look clean because the child is clean
self.assertEqual(set(), obj.obj_what_changed())
def test_initialize_objects(self):
class Foo(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('Bar')}
class Bar(base.CinderObject):
fields = {'foo': fields.StringField()}
obj = Foo()
self.assertEqual([], obj.objects)
self.assertEqual(set(), obj.obj_what_changed())
def test_obj_repr(self):
class Foo(base.ObjectListBase, base.CinderObject):
fields = {'objects': fields.ListOfObjectsField('Bar')}
class Bar(base.CinderObject):
fields = {'uuid': fields.StringField()}
obj = Foo(objects=[Bar(uuid='fake-uuid')])
self.assertEqual('Foo(objects=[Bar(fake-uuid)])', repr(obj))
class TestObjectSerializer(_BaseTestCase):
def test_serialize_entity_primitive(self):
ser = base.CinderObjectSerializer()
for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
self.assertEqual(thing, ser.serialize_entity(None, thing))
def test_deserialize_entity_primitive(self):
ser = base.CinderObjectSerializer()
for thing in (1, 'foo', [1, 2], {'foo': 'bar'}):
self.assertEqual(thing, ser.deserialize_entity(None, thing))
def _test_deserialize_entity_newer(self, obj_version, backported_to,
my_version='1.6'):
ser = base.CinderObjectSerializer()
ser._conductor = mock.Mock()
ser._conductor.object_backport.return_value = 'backported'
class MyTestObj(MyObj):
VERSION = my_version
obj = MyTestObj()
obj.VERSION = obj_version
primitive = obj.obj_to_primitive()
ser.deserialize_entity(self.context, primitive)
if backported_to is None:
self.assertFalse(ser._conductor.object_backport.called)
def test_deserialize_entity_newer_revision_does_not_backport_zero(self):
self._test_deserialize_entity_newer('1.6.0', None)
def test_deserialize_entity_newer_revision_does_not_backport(self):
self._test_deserialize_entity_newer('1.6.1', None)
def test_deserialize_dot_z_with_extra_stuff(self):
primitive = {'cinder_object.name': 'MyObj',
'cinder_object.namespace': 'cinder',
'cinder_object.version': '1.6.1',
'cinder_object.data': {
'foo': 1,
'unexpected_thing': 'foobar'}}
ser = base.CinderObjectSerializer()
obj = ser.deserialize_entity(self.context, primitive)
self.assertEqual(1, obj.foo)
self.assertFalse(hasattr(obj, 'unexpected_thing'))
# NOTE(danms): The serializer is where the logic lives that
# avoids backports for cases where only a .z difference in
# the received object version is detected. As a result, we
# end up with a version of what we expected, effectively the
# .0 of the object.
self.assertEqual('1.6', obj.VERSION)
def test_object_serialization(self):
ser = base.CinderObjectSerializer()
obj = MyObj()
primitive = ser.serialize_entity(self.context, obj)
self.assertIn('cinder_object.name', primitive)
obj2 = ser.deserialize_entity(self.context, primitive)
self.assertIsInstance(obj2, MyObj)
self.assertEqual(self.context, obj2._context)
def test_object_serialization_iterables(self):
ser = base.CinderObjectSerializer()
obj = MyObj()
for iterable in (list, tuple, set):
thing = iterable([obj])
primitive = ser.serialize_entity(self.context, thing)
self.assertEqual(1, len(primitive))
for item in primitive:
self.assertNotIsInstance(item, base.CinderObject)
thing2 = ser.deserialize_entity(self.context, primitive)
self.assertEqual(1, len(thing2))
for item in thing2:
self.assertIsInstance(item, MyObj)
# dict case
thing = {'key': obj}
primitive = ser.serialize_entity(self.context, thing)
self.assertEqual(1, len(primitive))
for item in primitive.itervalues():
self.assertNotIsInstance(item, base.CinderObject)
thing2 = ser.deserialize_entity(self.context, primitive)
self.assertEqual(1, len(thing2))
for item in thing2.itervalues():
self.assertIsInstance(item, MyObj)
# object-action updates dict case
thing = {'foo': obj.obj_to_primitive()}
primitive = ser.serialize_entity(self.context, thing)
self.assertEqual(thing, primitive)
thing2 = ser.deserialize_entity(self.context, thing)
self.assertIsInstance(thing2['foo'], base.CinderObject)

View File

@ -398,12 +398,13 @@ class TestCinderManageCmd(test.TestCase):
service_get_all.assert_called_once_with(mock.sentinel.ctxt)
self.assertEqual(expected_out, fake_out.getvalue())
@mock.patch('cinder.objects.base.CinderObjectSerializer')
@mock.patch('cinder.rpc.get_client')
@mock.patch('cinder.rpc.init')
@mock.patch('cinder.rpc.initialized', return_value=False)
@mock.patch('oslo.messaging.Target')
def test_volume_commands_init(self, messaging_target, rpc_initialized,
rpc_init, get_client):
rpc_init, get_client, object_serializer):
CONF.set_override('volume_topic', 'fake-topic')
mock_target = messaging_target.return_value
mock_rpc_client = get_client.return_value
@ -414,7 +415,8 @@ class TestCinderManageCmd(test.TestCase):
rpc_initialized.assert_called_once_with()
rpc_init.assert_called_once_with(CONF)
messaging_target.assert_called_once_with(topic='fake-topic')
get_client.assert_called_once_with(mock_target)
get_client.assert_called_once_with(mock_target,
serializer=object_serializer())
self.assertEqual(mock_rpc_client, rpc_client)
@mock.patch('cinder.db.volume_get')

View File

@ -1477,3 +1477,19 @@ class TestRetryDecorator(test.TestCase):
self.assertRaises(WrongException, raise_unexpected_error)
self.assertFalse(mock_sleep.called)
class VersionTestCase(test.TestCase):
def test_convert_version_to_int(self):
self.assertEqual(utils.convert_version_to_int('6.2.0'), 6002000)
self.assertEqual(utils.convert_version_to_int((6, 4, 3)), 6004003)
self.assertEqual(utils.convert_version_to_int((5, )), 5)
self.assertRaises(exception.CinderException,
utils.convert_version_to_int, '5a.6b')
def test_convert_version_to_string(self):
self.assertEqual(utils.convert_version_to_str(6007000), '6.7.0')
self.assertEqual(utils.convert_version_to_str(4), '4')
def test_convert_version_to_tuple(self):
self.assertEqual(utils.convert_version_to_tuple('6.7.0'), (6, 7, 0))

View File

@ -781,3 +781,29 @@ def retry(exceptions, interval=1, retries=3, backoff_rate=2):
return _wrapper
return _decorator
def convert_version_to_int(version):
try:
if isinstance(version, six.string_types):
version = convert_version_to_tuple(version)
if isinstance(version, tuple):
return reduce(lambda x, y: (x * 1000) + y, version)
except Exception:
msg = _("Version %s is invalid.") % version
raise exception.CinderException(msg)
def convert_version_to_str(version_int):
version_numbers = []
factor = 1000
while version_int != 0:
version_number = version_int - (version_int // factor * factor)
version_numbers.insert(0, six.text_type(version_number))
version_int = version_int / factor
return reduce(lambda x, y: "%s.%s" % (x, y), version_numbers)
def convert_version_to_tuple(version_str):
return tuple(int(part) for part in version_str.split('.'))

View File

@ -20,6 +20,7 @@ from oslo import messaging
from oslo_config import cfg
from oslo_serialization import jsonutils
from cinder.objects import base as objects_base
from cinder import rpc
from cinder.volume import utils
@ -66,7 +67,8 @@ class VolumeAPI(object):
super(VolumeAPI, self).__init__()
target = messaging.Target(topic=CONF.volume_topic,
version=self.BASE_RPC_API_VERSION)
self.client = rpc.get_client(target, '1.19')
serializer = objects_base.CinderObjectSerializer()
self.client = rpc.get_client(target, '1.19', serializer=serializer)
def create_consistencygroup(self, ctxt, group, host):
new_host = utils.extract_host(host)

View File

@ -46,6 +46,12 @@ ignore_messages = ["An attribute affected in cinder.tests",
# checked elsewhere. We also ignore cinder.tests for now due to high false
# positive rate.
ignore_modules = ["cinder/openstack/common/", "cinder/tests/"]
# Note(thangp): E1101 should be ignored for only cinder.object modules.
# E1101 is error code related to accessing a non-existent member of an
# object, but should be ignored because the object member is created
# dynamically.
objects_ignore_codes = ["E1101"]
objects_ignore_modules = ["cinder/objects/"]
KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"
@ -101,6 +107,10 @@ class LintOutput(object):
return True
if any(msg in self.message for msg in ignore_messages):
return True
if (self.code in objects_ignore_codes and
any(self.filename.startswith(name)
for name in objects_ignore_modules)):
return True
return False
def key(self):