openstack/nova
Code Issues Proposed changes

Add nova-status upgrade check for consoles

Browse Source 
This will check if a deployment is currently using consoles and warns
the operator to set [workarounds]enable_consoleauth = True on their
console proxy host if they are performing a rolling upgrade which is
not yet complete.

Partial-Bug: #1798188

Change-Id: Idd6079ce4038d6f19966e98bcc61422b61b3636b
This commit is contained in:
melanie witt 2018-10-16 20:29:18 +00:00
parent 6256c60716
commit d2535b0261
4 changed files with 267 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/source/cli/nova-status.rst
View File

@ -121,6 +121,9 @@ Upgrade
**19.0.0 (Stein)**
* Checks for the Placement API are modified to require version 1.30.
* Checks are added for the **nova-consoleauth** service to warn and provide
additional instructions to set **[workarounds]enable_consoleauth = True**
while performing a live/rolling upgrade.
See Also
========

83
nova/cmd/status.py
View File

@ -575,6 +575,87 @@ class UpgradeCommands(object):
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
def _check_console_auths(self):
"""Checks for console usage and warns with info for rolling upgrade.
Iterates all cells checking to see if the nova-consoleauth service is
non-deleted/non-disabled and whether there are any console token auths
in that cell database. If there is a nova-consoleauth service being
used and no console token auths in the cell database, emit a warning
telling the user to set [workarounds]enable_consoleauth = True if they
are performing a rolling upgrade.
"""
# If we're using cells v1, we don't need to check if the workaround
# needs to be used because cells v1 always uses nova-consoleauth.
# If the operator has already enabled the workaround, we don't need
# to check anything.
if CONF.cells.enable or CONF.workarounds.enable_consoleauth:
return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
# We need to check cell0 for nova-consoleauth service records because
# it's possible a deployment could have services stored in the cell0
# database, if they've defaulted their [database]connection in
# nova.conf to cell0.
meta = MetaData(bind=db_session.get_api_engine())
cell_mappings = Table('cell_mappings', meta, autoload=True)
mappings = cell_mappings.select().execute().fetchall()
if not mappings:
# There are no cell mappings so we can't determine this, just
# return a warning. The cellsv2 check would have already failed
# on this.
msg = (_('Unable to check consoles without cell mappings.'))
return UpgradeCheckResult(UpgradeCheckCode.WARNING, msg)
ctxt = nova_context.get_admin_context()
# If we find a non-deleted, non-disabled nova-consoleauth service in
# any cell, we will assume the deployment is using consoles.
using_consoles = False
for mapping in mappings:
with nova_context.target_cell(ctxt, mapping) as cctxt:
# Check for any non-deleted, non-disabled nova-consoleauth
# service.
meta = MetaData(bind=db_session.get_engine(context=cctxt))
services = Table('services', meta, autoload=True)
consoleauth_service_record = (
select([services.c.id]).select_from(services).where(and_(
services.c.binary == 'nova-consoleauth',
services.c.deleted == 0,
services.c.disabled == false())).execute().first())
if consoleauth_service_record:
using_consoles = True
break
if using_consoles:
# If the deployment is using consoles, we can only be certain the
# upgrade is complete if each compute service is >= Rocky and
# supports storing console token auths in the database backend.
for mapping in mappings:
# Skip cell0 as no compute services should be in it.
if mapping['uuid'] == cell_mapping_obj.CellMapping.CELL0_UUID:
continue
# Get the minimum nova-compute service version in this
# cell.
with nova_context.target_cell(ctxt, mapping) as cctxt:
min_version = self._get_min_service_version(
cctxt, 'nova-compute')
# We could get None for the minimum version in the case of
# new install where there are no computes. If there are
# compute services, they should all have versions.
if min_version is not None and min_version < 35:
msg = _("One or more cells were found which have "
"nova-compute services older than Rocky. "
"Please set the "
"'[workarounds]enable_consoleauth' "
"configuration option to 'True' on your "
"console proxy host if you are performing a "
"rolling upgrade to enable consoles to "
"function during a partial upgrade.")
return UpgradeCheckResult(UpgradeCheckCode.WARNING,
msg)
return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
# The format of the check functions is to return an UpgradeCheckResult
# object with the appropriate UpgradeCheckCode and details set. If the
# check hits warnings or failures then those should be stored in the
@ -595,6 +676,8 @@ class UpgradeCommands(object):
(_('API Service Version'), _check_api_service_version),
# Added in Rocky
(_('Request Spec Migration'), _check_request_spec_migration),
# Added in Stein (but also useful going back to Rocky)
(_('Console Auths'), _check_console_auths),
)
def _get_details(self, upgrade_check_result):

174
nova/tests/unit/cmd/test_status.py
View File

@ -1040,3 +1040,177 @@ class TestUpgradeCheckRequestSpecMigration(test.NoDBTestCase):
"'nova-manage db online_data_migrations' on each cell "
"to create the missing request specs." %
self.cell_mappings['cell2'].uuid, result.details)
class TestUpgradeCheckConsoles(test.NoDBTestCase):
"""Tests for the nova-status upgrade check for consoles."""
# We'll setup the database ourselves because we need to use cells fixtures
# for multiple cell mappings.
USES_DB_SELF = True
# This will create three cell mappings: cell0, cell1 (default) and cell2
NUMBER_OF_CELLS = 2
def setUp(self):
super(TestUpgradeCheckConsoles, self).setUp()
self.output = StringIO()
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
# We always need the API DB to be setup.
self.useFixture(nova_fixtures.Database(database='api'))
self.cmd = status.UpgradeCommands()
@staticmethod
def _create_service_in_cell(ctxt, cell, binary, is_deleted=False,
disabled=False, version=None,
create_token_auth=False):
with context.target_cell(ctxt, cell) as cctxt:
service = objects.Service(context=cctxt, binary=binary,
disabled=disabled, host='dontcare')
if version:
service.version = version
service.create()
if is_deleted:
service.destroy()
if create_token_auth:
# We have to create an instance in order to create a token
# auth.
inst = objects.Instance(context=cctxt,
uuid=uuidutils.generate_uuid())
inst.create()
auth = objects.ConsoleAuthToken(context=cctxt,
console_type='novnc',
host='hostname', port=6080,
instance_uuid=inst.uuid)
auth.authorize(CONF.consoleauth.token_ttl)
return service
def test_check_cells_v1_enabled(self):
"""This is a 'success' case since the console auths check is
ignored when running cells v1.
"""
self.flags(enable=True, group='cells')
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
def test_check_workaround_enabled(self):
"""This is a 'success' case since the console auths check is
ignored when the workaround is already enabled.
"""
self.flags(enable_consoleauth=True, group='workarounds')
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
def test_deleted_disabled_consoleauth(self):
"""Tests that services other than nova-consoleauth and deleted/disabled
nova-consoleauth services are filtered out.
"""
self._setup_cells()
ctxt = context.get_admin_context()
# Create a compute service in cell1.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-compute')
# Create a deleted consoleauth service in cell1.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-consoleauth', is_deleted=True)
# Create a compute service in cell2.
self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
'nova-compute')
# Create a disabled consoleauth service in cell2.
self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
'nova-consoleauth', disabled=True)
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
def test_consoleauth_with_upgrade_not_started(self):
"""Tests the scenario where the deployment is using consoles but has no
compute services >= Rocky, i.e. a not started upgrade.
"""
self._setup_cells()
ctxt = context.get_admin_context()
# Create a deleted consoleauth service in cell1.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-consoleauth', is_deleted=True)
# Create a live consoleauth service in cell0. (Asserts we check cell0).
self._create_service_in_cell(ctxt, self.cell_mappings['cell0'],
'nova-consoleauth')
# Create Queens compute services in the cells.
for cell in ['cell1', 'cell2']:
self._create_service_in_cell(ctxt, self.cell_mappings[cell],
'nova-compute', version=30)
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.WARNING, result.code)
def test_consoleauth_with_upgrade_complete(self):
"""Tests the scenario where the deployment is using consoles and has
all compute services >= Rocky in every cell database, i.e. a completed
upgrade.
"""
self._setup_cells()
ctxt = context.get_admin_context()
# Create a live consoleauth service in cell1 with token auth.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-consoleauth',
create_token_auth=True)
# Create a live consoleauth service in cell2 with token auth.
self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
'nova-consoleauth',
create_token_auth=True)
# Create Rocky compute services in the cells.
for cell in ['cell1', 'cell2']:
self._create_service_in_cell(ctxt, self.cell_mappings[cell],
'nova-compute', version=35)
# Create a Queens compute service in cell0. This not actually valid,
# we do it to assert that we skip cell0 when checking service versions.
self._create_service_in_cell(ctxt, self.cell_mappings['cell0'],
'nova-compute', version=30)
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
def test_consoleauth_with_upgrade_partial(self):
"""Tests the scenario where the deployment is using consoles and has
compute services >= Rocky in at least one, but not all, cell databases,
i.e. a partial upgrade.
"""
self._setup_cells()
ctxt = context.get_admin_context()
# Create a live consoleauth service in cell1.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-consoleauth')
# Create a live consoleauth service in cell2 with token auth.
self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
'nova-consoleauth',
create_token_auth=True)
# Create a Queens compute service in cell1.
self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
'nova-compute', version=30)
# Create a Rocky compute service in cell2.
self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
'nova-compute', version=35)
result = self.cmd._check_console_auths()
self.assertEqual(status.UpgradeCheckCode.WARNING, result.code)
self.assertIn("One or more cells were found which have nova-compute "
"services older than Rocky. "
"Please set the '[workarounds]enable_consoleauth' "
"configuration option to 'True' on your console proxy "
"host if you are performing a rolling upgrade to enable "
"consoles to function during a partial upgrade.",
result.details)

7
releasenotes/notes/nova-status-check-consoleauths-618acb3a67f97418.yaml Normal file
View File

@ -0,0 +1,7 @@
---
upgrade:
- |
A new check is added to the ``nova-status upgrade check`` CLI to check for
use of the ``nova-consoleauth`` service to warn and provide additional
instructions to set ``[workarounds]enable_consoleauth = True`` while
performing a live/rolling upgrade.