openstack/cinder
Code Issues Proposed changes

Avoid second restart on offline upgrades

Browse Source 
On offline upgrades, due to the rolling upgrade mechanism we need to
restart the cinder services twice to complete the upgrade, just like in
the rolling upgrade case.

The current offline upgrade process is:

- Stop cinder services
- Upgrade the cinder nodes
- Sync your DB
- Start the cinder services
- Restart all the cinder services

This second restart creates a bad user experience and it should not be
necessary on an offline upgrade, so this patch adds a new optional
parameter -called "--bump-versions"- to the cinder-manage db sync
command that allows us to skip the restart of the services.

Closes-Bug: #1756321
Change-Id: I1b58c637f6b2187a78c9c00a6c4933335439ad6f
This commit is contained in:
Gorka Eguileor 2018-03-16 13:11:59 +01:00
parent 4fb3b64549
commit 3cd2ebd375
4 changed files with 89 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cinder/cmd/manage.py
View File

@ -67,6 +67,7 @@ from oslo_log import log as logging
from oslo_utils import timeutils
# Need to register global_opts
from cinder.backup import rpcapi as backup_rpcapi
from cinder.common import config # noqa
from cinder.common import constants
from cinder import context
@ -77,7 +78,9 @@ from cinder.db.sqlalchemy import models
from cinder import exception
from cinder.i18n import _
from cinder import objects
from cinder.objects import base as ovo_base
from cinder import rpc
from cinder.scheduler import rpcapi as scheduler_rpcapi
from cinder import version
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import utils as vutils
@ -85,6 +88,14 @@ from cinder.volume import utils as vutils
CONF = cfg.CONF
RPC_VERSIONS = {
'cinder-scheduler': scheduler_rpcapi.SchedulerAPI.RPC_API_VERSION,
'cinder-volume': volume_rpcapi.VolumeAPI.RPC_API_VERSION,
'cinder-backup': backup_rpcapi.BackupAPI.RPC_API_VERSION,
}
OVO_VERSION = ovo_base.OBJ_VERSIONS.get_current()
def _get_non_shared_target_hosts(ctxt):
hosts = []
@ -266,18 +277,40 @@ class DbCommands(object):
@args('version', nargs='?', default=None, type=int,
help='Database version')
def sync(self, version=None):
@args('--bump-versions', dest='bump_versions', default=False,
action='store_true',
help='Update RPC and Objects versions when doing offline upgrades, '
'with this we no longer need to restart the services twice '
'after the upgrade to prevent ServiceTooOld exceptions.')
def sync(self, version=None, bump_versions=False):
"""Sync the database up to the most recent version."""
if version is not None and version > db.MAX_INT:
print(_('Version should be less than or equal to '
'%(max_version)d.') % {'max_version': db.MAX_INT})
sys.exit(1)
try:
return db_migration.db_sync(version)
result = db_migration.db_sync(version)
except db_exc.DBMigrationError as ex:
print("Error during database migration: %s" % ex)
sys.exit(1)
try:
if bump_versions:
ctxt = context.get_admin_context()
services = objects.ServiceList.get_all(ctxt)
for service in services:
rpc_version = RPC_VERSIONS[service.binary]
if (service.rpc_current_version != rpc_version or
service.object_current_version != OVO_VERSION):
service.rpc_current_version = rpc_version
service.object_current_version = OVO_VERSION
service.save()
except Exception as ex:
print(_('Error during service version bump: %s') % ex)
sys.exit(2)
return result
def version(self):
"""Print the current database version."""
print(migration.db_version(db_api.get_engine(),

29
cinder/tests/unit/test_cmd.py
View File

@ -356,12 +356,39 @@ class TestCinderManageCmd(test.TestCase):
ex = self.assertRaises(SystemExit, db_cmds.purge, age_in_days)
self.assertEqual(1, ex.code)
@mock.patch('cinder.objects.ServiceList.get_all')
@mock.patch('cinder.db.migration.db_sync')
def test_db_commands_sync(self, db_sync):
def test_db_commands_sync(self, db_sync, service_get_mock):
version = 11
db_cmds = cinder_manage.DbCommands()
db_cmds.sync(version=version)
db_sync.assert_called_once_with(version)
service_get_mock.assert_not_called()
@mock.patch('cinder.objects.Service.save')
@mock.patch('cinder.objects.ServiceList.get_all')
@mock.patch('cinder.db.migration.db_sync')
def test_db_commands_sync_bump_versions(self, db_sync, service_get_mock,
service_save):
ctxt = context.get_admin_context()
services = [fake_service.fake_service_obj(ctxt,
binary='cinder-' + binary,
rpc_current_version='0.1',
object_current_version='0.2')
for binary in ('volume', 'scheduler', 'backup')]
service_get_mock.return_value = services
version = 11
db_cmds = cinder_manage.DbCommands()
db_cmds.sync(version=version, bump_versions=True)
db_sync.assert_called_once_with(version)
self.assertEqual(3, service_save.call_count)
for service in services:
self.assertEqual(cinder_manage.RPC_VERSIONS[service.binary],
service.rpc_current_version)
self.assertEqual(cinder_manage.OVO_VERSION,
service.object_current_version)
@mock.patch('oslo_db.sqlalchemy.migration.db_version')
def test_db_commands_version(self, db_version):

11
doc/source/man/cinder-manage.rst
View File

@ -50,10 +50,19 @@ Cinder Db
Print the current database version.
``cinder-manage db sync``
``cinder-manage db sync [--bump-versions] [version]``
Sync the database up to the most recent version. This is the standard way to create the db as well.
This command interprets the following options when it is invoked:
version Database version
--bump-versions Update RPC and Objects versions when doing offline
upgrades, with this we no longer need to restart the
services twice after the upgrade to prevent ServiceTooOld
exceptions.
``cinder-manage db purge [<number of days>]``
Purge database entries that are marked as deleted, that are older than the number of days specified.

16
releasenotes/notes/sync-bump-versions-a1e6f6359173892e.yaml Normal file
View File

@ -0,0 +1,16 @@
---
features:
- Cinder-manage DB sync command can now bump the RPC and Objects versions of
the services to avoid a second restart when doing offline upgrades.
upgrade:
- On offline upgrades, due to the rolling upgrade mechanism we need to
restart the cinder services twice to complete the installation just like in
the rolling upgrades case. First you stop the cinder services, then you
upgrade them, you sync your DB, then you start all the cinder services, and
then you restart them all. To avoid this last restart we can now instruct
the DB sync to bump the services after the migration is completed, the
command to do this is `cinder-manage db sync --bump-versions`
fixes:
- After an offline upgrade we had to restart all Cinder services twice, now
with the `cinder-manage db sync --bump-versions` command we can avoid the
second restart.