openstack
/
cinder
Code Issues Proposed changes

Use manifest to backport OVOs during upgrades 

To make objects that have other objects as fields compatible to an
earlier version, oslo versioned objects uses either a manifest passed to
obj_to_primitive or the object's obj_relationships mapping.

Which means that if we don't have any of those mechanisms in place our
rolling upgrades mechanism will fail whenever we try to backport a
Versioned Object that has set an ObjectField field because Oslo
Versioned Object will not know how to backport that related object.

This patch introduces the usage of manifests on backports when we are
doing rolling upgrades.

For the manifest, we use the data in our Objects History.  Which means
that as long as we keep history in OBJ_VERSIONS right we will not have
to create and worry about keeping lists' child_versions field or our
versioned object's obj_relationships for fields with types
ListOfObjectsField and ObjectField.

We also don't have to worry about cascade version bumping, as in
changing the List OVO version whenever the OVO it contains gets bumped,
or bumping our OVO whenever one of the related OVO fields is bumped.

Closes-Bug: #1571566
Change-Id: Ibc1a1257830c925c10696c0b5aedd5f471c538d0
This commit is contained in:
Gorka Eguileor 2016-04-14 20:52:36 +02:00
parent caee4d0459
commit f0d34b7d9b
17 changed files with 219 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cinder/exception.py
View File

@ -774,6 +774,11 @@ ObjectActionError = obj_exc.ObjectActionError
ObjectFieldInvalid = obj_exc.ObjectFieldInvalid
class CappedVersionUnknown(CinderException):
message = _('Unrecoverable Error: Versioned Objects in DB are capped to '
'unknown version %(version)s.')
class VolumeGroupNotFound(CinderException):
message = _('Unable to find Volume Group: %(vg_name)s')

3
cinder/objects/backup.py
View File

@ -162,9 +162,6 @@ class BackupList(base.ObjectListBase, base.CinderObject):
fields = {
'objects': fields.ListOfObjectsField('Backup'),
}
child_versions = {
'1.0': '1.0'
}
@base.remotable_classmethod
def get_all(cls, context, filters=None, marker=None, limit=None,

13
cinder/objects/base.py
View File

@ -411,6 +411,17 @@ class CinderObjectSerializer(base.VersionedObjectSerializer):
super(CinderObjectSerializer, self).__init__()
self.version_cap = version_cap
# NOTE(geguileo): During upgrades we will use a manifest to ensure that
# all objects are properly backported. This allows us to properly
# backport child objects to the right version even if parent version
# has not been bumped.
if not version_cap or version_cap == OBJ_VERSIONS.get_current():
self.manifest = None
else:
if version_cap not in OBJ_VERSIONS:
raise exception.CappedVersionUnknown(version=version_cap)
self.manifest = OBJ_VERSIONS[version_cap]
def _get_capped_obj_version(self, obj):
objname = obj.obj_name()
version_dict = OBJ_VERSIONS.get(self.version_cap, {})
@ -436,5 +447,5 @@ class CinderObjectSerializer(base.VersionedObjectSerializer):
callable(entity.obj_to_primitive)):
# NOTE(dulek): Backport outgoing object to the capped version.
backport_ver = self._get_capped_obj_version(entity)
entity = entity.obj_to_primitive(backport_ver)
entity = entity.obj_to_primitive(backport_ver, self.manifest)
return entity

3
cinder/objects/cgsnapshot.py
View File

@ -127,9 +127,6 @@ class CGSnapshotList(base.ObjectListBase, base.CinderObject):
fields = {
'objects': fields.ListOfObjectsField('CGSnapshot')
}
child_version = {
'1.0': '1.0'
}
@base.remotable_classmethod
def get_all(cls, context, filters=None):

4
cinder/objects/consistencygroup.py
View File

@ -151,10 +151,6 @@ class ConsistencyGroupList(base.ObjectListBase, base.CinderObject):
fields = {
'objects': fields.ListOfObjectsField('ConsistencyGroup')
}
child_version = {
'1.0': '1.0',
'1.1': '1.1',
}
@base.remotable_classmethod
def get_all(cls, context, filters=None, marker=None, limit=None,

4
cinder/objects/service.py
View File

@ -139,10 +139,6 @@ class ServiceList(base.ObjectListBase, base.CinderObject):
fields = {
'objects': fields.ListOfObjectsField('Service'),
}
child_versions = {
'1.0': '1.0',
'1.1': '1.2',
}
@base.remotable_classmethod
def get_all(cls, context, filters=None):

3
cinder/objects/snapshot.py
View File

@ -226,9 +226,6 @@ class SnapshotList(base.ObjectListBase, base.CinderObject):
fields = {
'objects': fields.ListOfObjectsField('Snapshot'),
}
child_versions = {
'1.0': '1.0'
}
@base.remotable_classmethod
def get_all(cls, context, search_opts, marker=None, limit=None,

5
cinder/objects/volume.py
View File

@ -441,11 +441,6 @@ class VolumeList(base.ObjectListBase, base.CinderObject):
'objects': fields.ListOfObjectsField('Volume'),
}
child_versions = {
'1.0': '1.0',
'1.1': '1.1',
}
@classmethod
def _get_expected_attrs(cls, context):
expected_attrs = ['metadata', 'volume_type']

4
cinder/objects/volume_attachment.py
View File

@ -68,10 +68,6 @@ class VolumeAttachmentList(base.ObjectListBase, base.CinderObject):
'objects': fields.ListOfObjectsField('VolumeAttachment'),
}
child_versions = {
'1.0': '1.0',
}
@base.remotable_classmethod
def get_all_by_volume_id(cls, context, volume_id):
attachments = db.volume_attachment_get_all_by_volume_id(context,

5
cinder/objects/volume_type.py
View File

@ -107,11 +107,6 @@ class VolumeTypeList(base.ObjectListBase, base.CinderObject):
'objects': fields.ListOfObjectsField('VolumeType'),
}
child_versions = {
'1.0': '1.0',
'1.1': '1.0',
}
@base.remotable_classmethod
def get_all(cls, context, inactive=0, filters=None, marker=None,
limit=None, sort_keys=None, sort_dirs=None, offset=None):

8
cinder/tests/unit/backup/test_backup.py
View File

@ -301,19 +301,21 @@ class BackupTestCase(BaseBackupTest):
@mock.patch('cinder.objects.service.Service.get_minimum_obj_version')
@mock.patch('cinder.rpc.LAST_RPC_VERSIONS', {'cinder-backup': '1.3',
'cinder-volume': '1.7'})
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-backup': '1.5',
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-backup': '1.2',
'cinder-volume': '1.4'})
def test_reset(self, get_min_obj, get_min_rpc):
get_min_obj.return_value = 'liberty'
backup_mgr = manager.BackupManager()
backup_rpcapi = backup_mgr.backup_rpcapi
volume_rpcapi = backup_mgr.volume_rpcapi
self.assertEqual('1.3', backup_rpcapi.client.version_cap)
self.assertEqual('1.5',
self.assertEqual('1.2',
backup_rpcapi.client.serializer._base.version_cap)
self.assertEqual('1.7', volume_rpcapi.client.version_cap)
self.assertEqual('1.4',
volume_rpcapi.client.serializer._base.version_cap)
get_min_obj.return_value = objects.base.OBJ_VERSIONS.get_current()
backup_mgr.reset()
backup_rpcapi = backup_mgr.backup_rpcapi
@ -322,10 +324,12 @@ class BackupTestCase(BaseBackupTest):
backup_rpcapi.client.version_cap)
self.assertEqual(get_min_obj.return_value,
backup_rpcapi.client.serializer._base.version_cap)
self.assertIsNone(backup_rpcapi.client.serializer._base.manifest)
self.assertEqual(get_min_rpc.return_value,
volume_rpcapi.client.version_cap)
self.assertEqual(get_min_obj.return_value,
volume_rpcapi.client.serializer._base.version_cap)
self.assertIsNone(volume_rpcapi.client.serializer._base.manifest)
def test_is_working(self):
self.assertTrue(self.backup_mgr.is_working())

81
cinder/tests/unit/fake_objects.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2016 Red Hat Inc.
# Copyright (c) 2016 Intel Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import versionutils
from cinder import objects
@objects.base.CinderObjectRegistry.register_if(False)
class ChildObject(objects.base.CinderObject):
VERSION = '1.2'
fields = {
'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
'uuid': objects.base.fields.UUIDField(),
'text': objects.base.fields.StringField(nullable=True),
'integer': objects.base.fields.IntegerField(nullable=True),
}
def obj_make_compatible(self, primitive, target_version):
super(ChildObject, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
if target_version < (1, 1):
primitive.pop('text', None)
if target_version < (1, 2):
primitive.pop('integer', None)
@objects.base.CinderObjectRegistry.register_if(False)
class ParentObject(objects.base.CinderObject):
VERSION = '1.1'
fields = {
'uuid': objects.base.fields.UUIDField(),
'child': objects.base.fields.ObjectField('ChildObject', nullable=True),
'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
}
def obj_make_compatible(self, primitive, target_version):
super(ParentObject, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
if target_version < (1, 1):
primitive.pop('scheduled_at', None)
@objects.base.CinderObjectRegistry.register_if(False)
class ParentObjectList(objects.base.CinderObject, objects.base.ObjectListBase):
VERSION = ParentObject.VERSION
fields = {
'objects': objects.base.fields.ListOfObjectsField('ParentObject'),
}
class MyHistory(objects.base.CinderObjectVersionsHistory):
linked_objects = {'ParentObject': 'ParentObjectList'}
def __init__(self):
self.versions = ['1.0']
self['1.0'] = {'ChildObject': '1.0'}
self.add('1.1', {'ChildObject': '1.1'})
self.add('1.2', {'ParentObject': '1.0'})
self.add('1.3', {'ParentObjectList': '1.0'})
self.add('1.4', {'ParentObject': '1.1'})
self.add('1.5', {'ParentObjectList': '1.1'})
self.add('1.6', {'ChildObject': '1.2'})

128
cinder/tests/unit/objects/test_base.py
View File

@ -17,7 +17,6 @@ import uuid
from iso8601 import iso8601
import mock
from oslo_utils import versionutils
from oslo_versionedobjects import fields
from sqlalchemy import sql
@ -28,33 +27,16 @@ from cinder import exception
from cinder import objects
from cinder import test
from cinder.tests.unit import fake_constants as fake
from cinder.tests.unit import fake_objects
from cinder.tests.unit import objects as test_objects
@objects.base.CinderObjectRegistry.register_if(False)
class TestObject(objects.base.CinderObject):
VERSION = '1.1'
fields = {
'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
'uuid': objects.base.fields.UUIDField(),
'text': objects.base.fields.StringField(nullable=True),
}
def obj_make_compatible(self, primitive, target_version):
super(TestObject, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
if target_version < (1, 1):
primitive.pop('text', None)
class TestCinderObject(test_objects.BaseObjectsTestCase):
"""Tests methods from CinderObject."""
def setUp(self):
super(TestCinderObject, self).setUp()
self.obj = TestObject(
self.obj = fake_objects.ChildObject(
scheduled_at=None,
uuid=uuid.uuid4(),
text='text')
@ -720,21 +702,107 @@ class TestCinderDictObject(test_objects.BaseObjectsTestCase):
self.assertFalse('def' in obj)
@mock.patch('cinder.objects.base.OBJ_VERSIONS', {'1.0': {'TestObject': '1.0'},
'1.1': {'TestObject': '1.1'},
})
@mock.patch('cinder.objects.base.OBJ_VERSIONS', fake_objects.MyHistory())
class TestCinderObjectSerializer(test_objects.BaseObjectsTestCase):
def setUp(self):
super(TestCinderObjectSerializer, self).setUp()
self.obj = TestObject(scheduled_at=None, uuid=uuid.uuid4(),
text='text')
self.obj = fake_objects.ChildObject(scheduled_at=None,
uuid=uuid.uuid4(),
text='text',
integer=1)
self.parent = fake_objects.ParentObject(uuid=uuid.uuid4(),
child=self.obj,
scheduled_at=None)
self.parent_list = fake_objects.ParentObjectList(objects=[self.parent])
def test_serialize_entity_backport(self):
serializer = objects.base.CinderObjectSerializer('1.0')
primitive = serializer.serialize_entity(self.context, self.obj)
self.assertEqual('1.0', primitive['versioned_object.version'])
def test_serialize_init_current_has_no_manifest(self):
"""Test that pinned to current version we have no manifest."""
serializer = objects.base.CinderObjectSerializer('1.6')
# Serializer should not have a manifest
self.assertIsNone(serializer.manifest)
def test_serialize_init_no_cap_has_no_manifest(self):
"""Test that without cap we have no manifest."""
serializer = objects.base.CinderObjectSerializer()
# Serializer should not have a manifest
self.assertIsNone(serializer.manifest)
def test_serialize_init_pinned_has_manifest(self):
"""Test that pinned to older version we have manifest."""
objs_version = '1.5'
serializer = objects.base.CinderObjectSerializer(objs_version)
# Serializer should have the right manifest
self.assertDictEqual(fake_objects.MyHistory()[objs_version],
serializer.manifest)
def test_serialize_entity_unknown_version(self):
serializer = objects.base.CinderObjectSerializer('0.9')
"""Test that bad cap version will prevent serializer creation."""
self.assertRaises(exception.CappedVersionUnknown,
objects.base.CinderObjectSerializer, '0.9')
def test_serialize_entity_basic_no_backport(self):
"""Test single element serializer with no backport."""
serializer = objects.base.CinderObjectSerializer('1.6')
primitive = serializer.serialize_entity(self.context, self.obj)
self.assertEqual('1.2', primitive['versioned_object.version'])
data = primitive['versioned_object.data']
self.assertEqual(1, data['integer'])
self.assertEqual('text', data['text'])
def test_serialize_entity_basic_backport(self):
"""Test single element serializer with backport."""
serializer = objects.base.CinderObjectSerializer('1.5')
primitive = serializer.serialize_entity(self.context, self.obj)
self.assertEqual('1.1', primitive['versioned_object.version'])
data = primitive['versioned_object.data']
self.assertNotIn('integer', data)
self.assertEqual('text', data['text'])
def test_serialize_entity_full_no_backport(self):
"""Test related elements serialization with no backport."""
serializer = objects.base.CinderObjectSerializer('1.6')
primitive = serializer.serialize_entity(self.context, self.parent_list)
self.assertEqual('1.1', primitive['versioned_object.version'])
parent = primitive['versioned_object.data']['objects'][0]
self.assertEqual('1.1', parent['versioned_object.version'])
child = parent['versioned_object.data']['child']
self.assertEqual('1.2', child['versioned_object.version'])
def test_serialize_entity_full_backport_last_children(self):
"""Test related elements serialization with backport of the last child.
Test that using the manifest we properly backport a child object even
when all its parents have not changed their version.
"""
serializer = objects.base.CinderObjectSerializer('1.5')
primitive = serializer.serialize_entity(self.context, self.parent_list)
self.assertEqual('1.1', primitive['versioned_object.version'])
parent = primitive['versioned_object.data']['objects'][0]
self.assertEqual('1.1', parent['versioned_object.version'])
# Only the child has been backported
child = parent['versioned_object.data']['child']
self.assertEqual('1.1', child['versioned_object.version'])
# Check that the backport has been properly done
data = child['versioned_object.data']
self.assertNotIn('integer', data)
self.assertEqual('text', data['text'])
def test_serialize_entity_full_backport(self):
"""Test backport of the whole tree of related elements."""
serializer = objects.base.CinderObjectSerializer('1.3')
primitive = serializer.serialize_entity(self.context, self.parent_list)
# List has been backported
self.assertEqual('1.0', primitive['versioned_object.version'])
parent = primitive['versioned_object.data']['objects'][0]
# Parent has been backported as well
self.assertEqual('1.0', parent['versioned_object.version'])
# And the backport has been properly done
data = parent['versioned_object.data']
self.assertNotIn('scheduled_at', data)
# And child as well
child = parent['versioned_object.data']['child']
self.assertEqual('1.1', child['versioned_object.version'])
# Check that the backport has been properly done
data = child['versioned_object.data']
self.assertNotIn('integer', data)
self.assertEqual('text', data['text'])

12
cinder/tests/unit/objects/test_objects.py
View File

@ -25,21 +25,21 @@ from cinder import test
object_data = {
'Backup': '1.4-bcd1797dc2f3e17a46571525e9dbec30',
'BackupImport': '1.4-bcd1797dc2f3e17a46571525e9dbec30',
'BackupList': '1.0-24591dabe26d920ce0756fe64cd5f3aa',
'BackupList': '1.0-7350483276ddb74960c8c39b69192eaa',
'CGSnapshot': '1.0-de2586a31264d7647f40c762dece9d58',
'CGSnapshotList': '1.0-e8c3f4078cd0ee23487b34d173eec776',
'ConsistencyGroup': '1.2-de280886bd04d7e3184c1f7c3a7e2074',
'ConsistencyGroupList': '1.1-73916823b697dfa0c7f02508d87e0f28',
'Service': '1.3-66c8e1683f58546c54551e9ff0a3b111',
'ServiceList': '1.1-cb758b200f0a3a90efabfc5aa2ffb627',
'ServiceList': '1.1-07d2be494d704784ad2af4d4c91e68e5',
'Snapshot': '1.1-ac41f2fe2fb0e34127155d1ec6e4c7e0',
'SnapshotList': '1.0-71661e7180ef6cc51501704a9bea4bf1',
'SnapshotList': '1.0-58441afd20ddf9417b8879aa6de1ee6f',
'Volume': '1.3-049e3e5dc411b1a4deb7d6ee4f1ad5ef',
'VolumeList': '1.1-8859f973dc02e9eb4582063a171bd0f1',
'VolumeAttachment': '1.0-8fc9a9ac6f554fdf2a194d25dbf28a3b',
'VolumeAttachmentList': '1.0-307d2b6c8dd55ef854f6386898e9e98e',
'VolumeList': '1.1-03ba6cb8c546683e64e15c50042cb1a3',
'VolumeAttachmentList': '1.0-4ef79b3824e5d1717ebe0d0558ddff96',
'VolumeType': '1.0-dd980cfd1eef2dcce941a981eb469fc8',
'VolumeTypeList': '1.1-8a1016c03570dc13b9a33fe04a6acb2c',
'VolumeTypeList': '1.1-68a4549e98563caec82a2018638fa69c',
}

7
cinder/tests/unit/scheduler/test_scheduler.py
View File

@ -24,6 +24,7 @@ from cinder import context
from cinder import db
from cinder import exception
from cinder.message import defined_messages
from cinder import objects
from cinder.objects import fields
from cinder.scheduler import driver
from cinder.scheduler import filter_scheduler
@ -74,14 +75,15 @@ class SchedulerManagerTestCase(test.TestCase):
@mock.patch('cinder.objects.service.Service.get_minimum_rpc_version')
@mock.patch('cinder.objects.service.Service.get_minimum_obj_version')
@mock.patch('cinder.rpc.LAST_RPC_VERSIONS', {'cinder-volume': '1.3'})
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-volume': '1.5'})
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-volume': '1.4'})
def test_reset(self, get_min_obj, get_min_rpc):
mgr = self.manager_cls()
volume_rpcapi = mgr.driver.volume_rpcapi
self.assertEqual('1.3', volume_rpcapi.client.version_cap)
self.assertEqual('1.5',
self.assertEqual('1.4',
volume_rpcapi.client.serializer._base.version_cap)
get_min_obj.return_value = objects.base.OBJ_VERSIONS.get_current()
mgr.reset()
volume_rpcapi = mgr.driver.volume_rpcapi
@ -89,6 +91,7 @@ class SchedulerManagerTestCase(test.TestCase):
volume_rpcapi.client.version_cap)
self.assertEqual(get_min_obj.return_value,
volume_rpcapi.client.serializer._base.version_cap)
self.assertIsNone(volume_rpcapi.client.serializer._base.manifest)
@mock.patch('cinder.scheduler.driver.Scheduler.'
'update_service_capabilities')

4
cinder/tests/unit/test_rpc.py
View File

@ -37,14 +37,14 @@ class RPCAPITestCase(test.TestCase):
@mock.patch('cinder.objects.Service.get_minimum_rpc_version',
return_value='1.2')
@mock.patch('cinder.objects.Service.get_minimum_obj_version',
return_value='1.7')
return_value='1.4')
@mock.patch('cinder.rpc.get_client')
def test_init(self, get_client, get_min_obj, get_min_rpc):
def fake_get_client(target, version_cap, serializer):
self.assertEqual(FakeAPI.TOPIC, target.topic)
self.assertEqual(FakeAPI.RPC_API_VERSION, target.version)
self.assertEqual('1.2', version_cap)
self.assertEqual('1.7', serializer.version_cap)
self.assertEqual('1.4', serializer.version_cap)
get_client.side_effect = fake_get_client
FakeAPI()

6
cinder/tests/unit/test_volume.py
View File

@ -505,14 +505,15 @@ class VolumeTestCase(BaseVolumeTestCase):
@mock.patch('cinder.objects.service.Service.get_minimum_rpc_version')
@mock.patch('cinder.objects.service.Service.get_minimum_obj_version')
@mock.patch('cinder.rpc.LAST_RPC_VERSIONS', {'cinder-scheduler': '1.3'})
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-scheduler': '1.5'})
@mock.patch('cinder.rpc.LAST_OBJ_VERSIONS', {'cinder-scheduler': '1.4'})
def test_reset(self, get_min_obj, get_min_rpc):
vol_mgr = vol_manager.VolumeManager()
scheduler_rpcapi = vol_mgr.scheduler_rpcapi
self.assertEqual('1.3', scheduler_rpcapi.client.version_cap)
self.assertEqual('1.5',
self.assertEqual('1.4',
scheduler_rpcapi.client.serializer._base.version_cap)
get_min_obj.return_value = objects.base.OBJ_VERSIONS.get_current()
vol_mgr.reset()
scheduler_rpcapi = vol_mgr.scheduler_rpcapi
@ -520,6 +521,7 @@ class VolumeTestCase(BaseVolumeTestCase):
scheduler_rpcapi.client.version_cap)
self.assertEqual(get_min_obj.return_value,
scheduler_rpcapi.client.serializer._base.version_cap)
self.assertIsNone(scheduler_rpcapi.client.serializer._base.manifest)
@mock.patch.object(vol_manager.VolumeManager,
'update_service_capabilities')