Merge "Add RPC and object version pinning"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
a84c2e0d8e
|
|
@ -47,11 +47,24 @@ Things to do before releasing
|
|||
TEMPEST_BAREMETAL_MAX_MICROVERSION in devstack/lib/ironic to make sure that
|
||||
unsupported API tempest tests are skipped on stable branches.
|
||||
|
||||
* To support rolling upgrades, add this new release version (and release name
|
||||
if it is a named release) into ironic/common/release_mappings.py:
|
||||
|
||||
* in RELEASE_MAPPING, make a copy of the 'master' entry, and rename the first
|
||||
'master' entry to the new semver release version.
|
||||
* If this is a named release, add a RELEASE_MAPPING entry for the named
|
||||
release. Its value should be the same as that of the latest semver one
|
||||
(that you just added above).
|
||||
* Regenerate the sample config file, so that the choices for the
|
||||
``[DEFAULT]/pin_release_version`` configuration are accurate.
|
||||
|
||||
.. _`standards`: http://docs.openstack.org/developer/ironic/dev/faq.html#know-if-a-release-note-is-needed-for-my-change
|
||||
|
||||
Things to do after releasing
|
||||
============================
|
||||
|
||||
When a release is done that results in a stable branch
|
||||
------------------------------------------------------
|
||||
When a release is done that results in a stable branch for the project, the
|
||||
release automation will push a number of changes that need to be approved.
|
||||
|
||||
|
|
@ -82,8 +95,21 @@ Additionally, changes need to be made on master to:
|
|||
and `pbr documentation
|
||||
<http://docs.openstack.org/developer/pbr/#version>`_ for details.
|
||||
|
||||
For all releases
|
||||
----------------
|
||||
For all releases, whether or not it results in a stable branch:
|
||||
|
||||
* update the specs repo to mark any specs completed in the release as
|
||||
implemented.
|
||||
|
||||
* remove any -2s on patches that were blocked until after the release.
|
||||
|
||||
* to support rolling upgrades, make these changes in
|
||||
ironic/common/release_mappings.py:
|
||||
|
||||
* if the release was a named release, delete any entries from
|
||||
RELEASE_MAPPING associated with the oldest named release. Since we
|
||||
support upgrades between adjacent named releases, the master branch will
|
||||
only support upgrades from the most recent named release to master.
|
||||
* regenerate the sample config file, so that the choices for the
|
||||
``[DEFAULT]/pin_release_version`` configuration are accurate.
|
||||
|
|
|
|||
|
|
@ -354,6 +354,15 @@
|
|||
# value)
|
||||
#host = localhost
|
||||
|
||||
# Used for rolling upgrades. Setting this option downgrades
|
||||
# the internal ironic RPC communication to the specified
|
||||
# version to enable communication with older services. When
|
||||
# doing a rolling upgrade from version X to version Y, set
|
||||
# this to X. Defaults to using the newest possible RPC
|
||||
# behavior. (string value)
|
||||
# Allowed values: ocata, 7.0
|
||||
#pin_release_version = <None>
|
||||
|
||||
# Path to the rootwrap configuration file to use for running
|
||||
# commands as root. (string value)
|
||||
#rootwrap_config = /etc/ironic/rootwrap.conf
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
# Copyright 2016 Intel Corp.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# NOTE(xek): This decides the version cap of RPC messages sent to conductor
|
||||
# and objects during rolling upgrades, when [DEFAULT]/pin_release_version
|
||||
# configuration is set.
|
||||
#
|
||||
# Remember to add a new entry for the new version that is shipping in a new
|
||||
# release.
|
||||
#
|
||||
# We support a rolling upgrade between adjacent named releases, as well as
|
||||
# between a release and master, so old, unsupported releases can be removed,
|
||||
# together with the supporting code, which is typically found in an object's
|
||||
# make_compatible methods and RPC client code.
|
||||
|
||||
# NOTE(xek): The format of this dict is:
|
||||
# { '<release version>': {
|
||||
# 'rpc': '<RPC API version>',
|
||||
# 'objects': {
|
||||
# '<object class name>': '<object version>',
|
||||
# }
|
||||
# },
|
||||
# }
|
||||
# The list should contain all objects which are persisted in the database and
|
||||
# sent over RPC. Notifications/Payloads are not being included here since we
|
||||
# don't need to pin them during rolling upgrades.
|
||||
#
|
||||
# There should always be a 'master' entry that reflects the objects in the
|
||||
# master branch.
|
||||
#
|
||||
# Just before doing a release, copy the 'master' entry, and rename the first
|
||||
# 'master' entry to the (semver) version being released.
|
||||
#
|
||||
# Just after doing a named release, delete any entries associated with the
|
||||
# oldest named release.
|
||||
RELEASE_MAPPING = {
|
||||
'7.0': {
|
||||
'rpc': '1.40',
|
||||
'objects': {
|
||||
'Node': '1.21',
|
||||
'Conductor': '1.2',
|
||||
'Chassis': '1.3',
|
||||
'Port': '1.6',
|
||||
'Portgroup': '1.3',
|
||||
'VolumeConnector': '1.0',
|
||||
'VolumeTarget': '1.0',
|
||||
}
|
||||
},
|
||||
'master': {
|
||||
'rpc': '1.40',
|
||||
'objects': {
|
||||
'Node': '1.21',
|
||||
'Conductor': '1.2',
|
||||
'Chassis': '1.3',
|
||||
'Port': '1.6',
|
||||
'Portgroup': '1.3',
|
||||
'VolumeConnector': '1.0',
|
||||
'VolumeTarget': '1.0',
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
# NOTE(xek): Assign each named release to the appropriate semver.
|
||||
#
|
||||
# Just before we do a new named release, add a mapping for the new
|
||||
# named release.
|
||||
#
|
||||
# Just after we do a new named release, delete the oldest named
|
||||
# release (that we are no longer supporting for a rolling upgrade).
|
||||
#
|
||||
# There should be at most two named mappings here.
|
||||
RELEASE_MAPPING['ocata'] = RELEASE_MAPPING['7.0']
|
||||
|
||||
# List of available versions with named versions first; 'master' is excluded.
|
||||
RELEASE_VERSIONS = sorted(set(RELEASE_MAPPING) - {'master'}, reverse=True)
|
||||
|
|
@ -25,6 +25,7 @@ import oslo_messaging as messaging
|
|||
from ironic.common import exception
|
||||
from ironic.common import hash_ring
|
||||
from ironic.common.i18n import _
|
||||
from ironic.common import release_mappings as versions
|
||||
from ironic.common import rpc
|
||||
from ironic.conductor import manager
|
||||
from ironic.conf import CONF
|
||||
|
|
@ -103,8 +104,10 @@ class ConductorAPI(object):
|
|||
target = messaging.Target(topic=self.topic,
|
||||
version='1.0')
|
||||
serializer = objects_base.IronicObjectSerializer()
|
||||
self.client = rpc.get_client(target,
|
||||
version_cap=self.RPC_API_VERSION,
|
||||
release_ver = versions.RELEASE_MAPPING.get(CONF.pin_release_version)
|
||||
version_cap = (release_ver['rpc'] if release_ver
|
||||
else self.RPC_API_VERSION)
|
||||
self.client = rpc.get_client(target, version_cap=version_cap,
|
||||
serializer=serializer)
|
||||
# NOTE(deva): this is going to be buggy
|
||||
self.ring_manager = hash_ring.HashRingManager()
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from oslo_config import cfg
|
|||
from oslo_utils import netutils
|
||||
|
||||
from ironic.common.i18n import _
|
||||
from ironic.common import release_mappings as versions
|
||||
|
||||
|
||||
_ENABLED_IFACE_HELP = _('Specify the list of {0} interfaces to load during '
|
||||
|
|
@ -261,6 +262,15 @@ service_opts = [
|
|||
'However, the node name must be valid within '
|
||||
'an AMQP key, and if using ZeroMQ, a valid '
|
||||
'hostname, FQDN, or IP address.')),
|
||||
cfg.StrOpt('pin_release_version',
|
||||
choices=versions.RELEASE_VERSIONS,
|
||||
# TODO(xek): mutable=True,
|
||||
help=_('Used for rolling upgrades. Setting this option '
|
||||
'downgrades the internal ironic RPC communication to '
|
||||
'the specified version to enable communication with '
|
||||
'older services. When doing a rolling upgrade from '
|
||||
'version X to version Y, set this to X. Defaults to '
|
||||
'using the newest possible RPC behavior.')),
|
||||
]
|
||||
|
||||
utils_opts = [
|
||||
|
|
|
|||
|
|
@ -14,12 +14,17 @@
|
|||
|
||||
"""Ironic common internal object model"""
|
||||
|
||||
from oslo_log import log
|
||||
from oslo_utils import versionutils
|
||||
from oslo_versionedobjects import base as object_base
|
||||
|
||||
from ironic.common import release_mappings as versions
|
||||
from ironic.conf import CONF
|
||||
from ironic import objects
|
||||
from ironic.objects import fields as object_fields
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class IronicObjectRegistry(object_base.VersionedObjectRegistry):
|
||||
def registration_hook(self, cls, index):
|
||||
|
|
@ -103,7 +108,35 @@ class IronicObject(object_base.VersionedObject):
|
|||
return [cls._from_db_object(cls(context), db_obj)
|
||||
for db_obj in db_objects]
|
||||
|
||||
def _get_target_version(self):
|
||||
"""Returns the target version for this object.
|
||||
|
||||
If pinned, returns the version of this object corresponding to
|
||||
the pin. Otherwise, returns this (latest) version of the object.
|
||||
"""
|
||||
pinned_version = None
|
||||
pin = CONF.pin_release_version
|
||||
if pin:
|
||||
version_manifest = versions.RELEASE_MAPPING[pin]['objects']
|
||||
pinned_version = version_manifest.get(self.obj_name())
|
||||
return pinned_version or self.__class__.VERSION
|
||||
|
||||
|
||||
class IronicObjectSerializer(object_base.VersionedObjectSerializer):
|
||||
# Base class to use for object hydration
|
||||
OBJ_BASE_CLASS = IronicObject
|
||||
|
||||
def serialize_entity(self, context, entity):
|
||||
if isinstance(entity, (tuple, list, set, dict)):
|
||||
entity = self._process_iterable(context, self.serialize_entity,
|
||||
entity)
|
||||
elif (hasattr(entity, 'obj_to_primitive')
|
||||
and callable(entity.obj_to_primitive)):
|
||||
target_version = entity._get_target_version()
|
||||
# NOTE(xek): If the version is pinned, target_version is an older
|
||||
# object version and entity's obj_make_compatible method is called
|
||||
# to backport the object before serialization.
|
||||
entity = entity.obj_to_primitive(target_version=target_version)
|
||||
elif not isinstance(entity, (int, str, bool, float, type)) and entity:
|
||||
LOG.warning("Entity %s was not serialized.", str(entity))
|
||||
return entity
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
# Copyright 2016 Intel Corp.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_utils import versionutils
|
||||
import six
|
||||
|
||||
from ironic.common.release_mappings import RELEASE_MAPPING
|
||||
from ironic.conductor import rpcapi
|
||||
from ironic.db.sqlalchemy import models
|
||||
from ironic.objects import base as obj_base
|
||||
from ironic.tests import base
|
||||
from ironic import version
|
||||
|
||||
|
||||
def _check_versions_compatibility(conf_version, actual_version):
|
||||
"""Checks the configured version against the actual version.
|
||||
|
||||
Returns True if the configured version is <= the actual version;
|
||||
otherwise returns False.
|
||||
|
||||
param conf_version: configured version, a string with dots
|
||||
param actual_version: actual version, a string with dots
|
||||
"""
|
||||
conf_cap = versionutils.convert_version_to_tuple(conf_version)
|
||||
actual_cap = versionutils.convert_version_to_tuple(actual_version)
|
||||
return conf_cap <= actual_cap
|
||||
|
||||
|
||||
class ReleaseMappingsTestCase(base.TestCase):
|
||||
"""Tests whether the dict RELEASE_MAPPING is correct, valid and consistent.
|
||||
|
||||
"""
|
||||
def test_structure(self):
|
||||
for value in RELEASE_MAPPING.values():
|
||||
self.assertTrue(isinstance(value, dict))
|
||||
self.assertEqual({'rpc', 'objects'}, set(value))
|
||||
self.assertTrue(isinstance(value['rpc'], six.string_types))
|
||||
self.assertTrue(isinstance(value['objects'], dict))
|
||||
for obj_value in value['objects'].values():
|
||||
self.assertTrue(isinstance(obj_value, six.string_types))
|
||||
tuple_ver = versionutils.convert_version_to_tuple(obj_value)
|
||||
self.assertEqual(2, len(tuple_ver))
|
||||
|
||||
def test_object_names_are_registered(self):
|
||||
registered_objects = set(obj_base.IronicObjectRegistry.obj_classes())
|
||||
for mapping in RELEASE_MAPPING.values():
|
||||
objects = set(mapping['objects'])
|
||||
self.assertTrue(objects.issubset(registered_objects))
|
||||
|
||||
def test_contains_current_release_entry(self):
|
||||
version_info = str(version.version_info)
|
||||
# NOTE(sborkows): We only need first two values from version (like 5.1)
|
||||
# and assume version_info is of the form 'x.y.z'.
|
||||
current_release = version_info[:version_info.rfind('.')]
|
||||
self.assertIn(current_release, RELEASE_MAPPING)
|
||||
|
||||
def test_current_rpc_version(self):
|
||||
self.assertEqual(rpcapi.ConductorAPI.RPC_API_VERSION,
|
||||
RELEASE_MAPPING['master']['rpc'])
|
||||
|
||||
def test_current_object_versions(self):
|
||||
registered_objects = obj_base.IronicObjectRegistry.obj_classes()
|
||||
for obj, objver in RELEASE_MAPPING['master']['objects'].items():
|
||||
self.assertEqual(registered_objects[obj][0].VERSION, objver)
|
||||
|
||||
def test_contains_all_db_objects(self):
|
||||
self.assertIn('master', RELEASE_MAPPING)
|
||||
model_names = set((s.__name__ for s in models.Base.__subclasses__()))
|
||||
exceptions = set(['NodeTag', 'ConductorHardwareInterfaces'])
|
||||
# NOTE(xek): As a rule, all models which can be changed between
|
||||
# releases or are sent through RPC should have their counterpart
|
||||
# versioned objects. This means all, but very simple models.
|
||||
model_names -= exceptions
|
||||
object_names = set(RELEASE_MAPPING['master']['objects'])
|
||||
self.assertEqual(model_names, object_names)
|
||||
|
||||
def test_rpc_and_objects_versions_supported(self):
|
||||
registered_objects = obj_base.IronicObjectRegistry.obj_classes()
|
||||
for versions in RELEASE_MAPPING.values():
|
||||
self.assertTrue(_check_versions_compatibility(
|
||||
versions['rpc'], rpcapi.ConductorAPI.RPC_API_VERSION))
|
||||
for obj_name, obj_ver in versions['objects'].items():
|
||||
self.assertTrue(_check_versions_compatibility(
|
||||
obj_ver, registered_objects[obj_name][0].VERSION))
|
||||
|
|
@ -27,6 +27,7 @@ from oslo_messaging import _utils as messaging_utils
|
|||
|
||||
from ironic.common import boot_devices
|
||||
from ironic.common import exception
|
||||
from ironic.common import release_mappings
|
||||
from ironic.common import states
|
||||
from ironic.conductor import manager as conductor_manager
|
||||
from ironic.conductor import rpcapi as conductor_rpcapi
|
||||
|
|
@ -45,6 +46,22 @@ class ConductorRPCAPITestCase(tests_base.TestCase):
|
|||
conductor_manager.ConductorManager.RPC_API_VERSION,
|
||||
conductor_rpcapi.ConductorAPI.RPC_API_VERSION)
|
||||
|
||||
@mock.patch('ironic.common.rpc.get_client')
|
||||
def test_version_cap(self, mock_get_client):
|
||||
conductor_rpcapi.ConductorAPI()
|
||||
self.assertEqual(conductor_rpcapi.ConductorAPI.RPC_API_VERSION,
|
||||
mock_get_client.call_args[1]['version_cap'])
|
||||
|
||||
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
|
||||
@mock.patch('ironic.common.rpc.get_client')
|
||||
def test_version_capped(self, mock_get_client, mock_release_mapping):
|
||||
CONF.set_override('pin_release_version',
|
||||
release_mappings.RELEASE_VERSIONS[0],
|
||||
enforce_type=True)
|
||||
mock_release_mapping.get.return_value = {'rpc': '3'}
|
||||
conductor_rpcapi.ConductorAPI()
|
||||
self.assertEqual('3', mock_get_client.call_args[1]['version_cap'])
|
||||
|
||||
|
||||
class RPCAPITestCase(base.DbTestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ from oslo_versionedobjects import fixture as object_fixture
|
|||
import six
|
||||
|
||||
from ironic.common import context
|
||||
from ironic.common import release_mappings
|
||||
from ironic.conf import CONF
|
||||
from ironic.objects import base
|
||||
from ironic.objects import fields
|
||||
from ironic.tests import base as test_base
|
||||
|
|
@ -37,6 +39,11 @@ class MyObj(base.IronicObject, object_base.VersionedObjectDictCompat):
|
|||
'missing': fields.StringField(),
|
||||
}
|
||||
|
||||
def obj_make_compatible(self, primitive, target_version):
|
||||
super(MyObj, self).obj_make_compatible(primitive, target_version)
|
||||
if target_version == '1.4' and 'missing' in primitive:
|
||||
del primitive['missing']
|
||||
|
||||
def obj_load_attr(self, attrname):
|
||||
setattr(self, attrname, 'loaded!')
|
||||
|
||||
|
|
@ -518,6 +525,90 @@ class TestObjectSerializer(test_base.TestCase):
|
|||
"Test object with unsupported (newer) version and revision"
|
||||
self._test_deserialize_entity_newer('1.7', '1.6.1', my_version='1.6.1')
|
||||
|
||||
@mock.patch.object(MyObj, 'obj_make_compatible')
|
||||
def test_serialize_entity_no_backport(self, make_compatible_mock):
|
||||
"""Test single element serializer with no backport."""
|
||||
serializer = base.IronicObjectSerializer()
|
||||
obj = MyObj(self.context)
|
||||
obj.foo = 1
|
||||
obj.bar = 'text'
|
||||
obj.missing = 'textt'
|
||||
primitive = serializer.serialize_entity(self.context, obj)
|
||||
self.assertEqual('1.5', primitive['ironic_object.version'])
|
||||
data = primitive['ironic_object.data']
|
||||
self.assertEqual(1, data['foo'])
|
||||
self.assertEqual('text', data['bar'])
|
||||
self.assertEqual('textt', data['missing'])
|
||||
changes = primitive['ironic_object.changes']
|
||||
self.assertEqual(set(['foo', 'bar', 'missing']), set(changes))
|
||||
make_compatible_mock.assert_not_called()
|
||||
|
||||
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
|
||||
def test_serialize_entity_backport(self, mock_release_mapping):
|
||||
"""Test single element serializer with backport."""
|
||||
CONF.set_override('pin_release_version',
|
||||
release_mappings.RELEASE_VERSIONS[-1],
|
||||
enforce_type=True)
|
||||
mock_release_mapping.__getitem__.return_value = {
|
||||
'objects': {
|
||||
'MyObj': '1.4',
|
||||
}
|
||||
}
|
||||
serializer = base.IronicObjectSerializer()
|
||||
obj = MyObj(self.context)
|
||||
obj.foo = 1
|
||||
obj.bar = 'text'
|
||||
obj.missing = 'textt'
|
||||
primitive = serializer.serialize_entity(self.context, obj)
|
||||
self.assertEqual('1.4', primitive['ironic_object.version'])
|
||||
data = primitive['ironic_object.data']
|
||||
self.assertEqual(1, data['foo'])
|
||||
self.assertEqual('text', data['bar'])
|
||||
self.assertNotIn('missing', data)
|
||||
changes = primitive['ironic_object.changes']
|
||||
self.assertEqual(set(['foo', 'bar']), set(changes))
|
||||
|
||||
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
|
||||
def test_serialize_entity_invalid_pin(self, mock_release_mapping):
|
||||
CONF.set_override('pin_release_version',
|
||||
release_mappings.RELEASE_VERSIONS[-1],
|
||||
enforce_type=True)
|
||||
mock_release_mapping.__getitem__.return_value = {
|
||||
'objects': {
|
||||
'MyObj': '1.6',
|
||||
}
|
||||
}
|
||||
serializer = base.IronicObjectSerializer()
|
||||
obj = MyObj(self.context)
|
||||
self.assertRaises(object_exception.InvalidTargetVersion,
|
||||
serializer.serialize_entity, self.context, obj)
|
||||
|
||||
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
|
||||
def test_serialize_entity_no_pin(self, mock_release_mapping):
|
||||
CONF.set_override('pin_release_version',
|
||||
release_mappings.RELEASE_VERSIONS[-1],
|
||||
enforce_type=True)
|
||||
mock_release_mapping.__getitem__.return_value = {
|
||||
'objects': {}
|
||||
}
|
||||
serializer = base.IronicObjectSerializer()
|
||||
obj = MyObj(self.context)
|
||||
primitive = serializer.serialize_entity(self.context, obj)
|
||||
self.assertEqual('1.5', primitive['ironic_object.version'])
|
||||
|
||||
@mock.patch('ironic.objects.base.IronicObject._get_target_version')
|
||||
@mock.patch('ironic.objects.base.LOG.warning')
|
||||
def test_serialize_entity_unknown_entity(self, mock_warn, mock_version):
|
||||
class Foo(object):
|
||||
fields = {'foobar': fields.IntegerField()}
|
||||
|
||||
serializer = base.IronicObjectSerializer()
|
||||
obj = Foo()
|
||||
primitive = serializer.serialize_entity(self.context, obj)
|
||||
self.assertEqual(obj, primitive)
|
||||
self.assertTrue(mock_warn.called)
|
||||
mock_version.assert_not_called()
|
||||
|
||||
|
||||
class TestRegistry(test_base.TestCase):
|
||||
@mock.patch('ironic.objects.base.objects')
|
||||
|
|
|
|||
Loading…
Reference in New Issue