openstack
/
nova
Code Issues Proposed changes

Merge "Warn when starting services with older than N-1 computes" into stable/victoria

This commit is contained in:
Zuul 2021-01-10 01:54:35 +00:00 committed by Gerrit Code Review
parent c70d9743a2 0c5ca351e2
commit cb4963ba8c
12 changed files with 242 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/source/contributor/ptl-guide.rst
View File

@ -257,6 +257,9 @@ Immediately after RC
* Example: https://review.opendev.org/543580
* Bump the oldest supported compute service version
* https://review.opendev.org/#/c/738482/
* Create the launchpad series for the next cycle
* Set the development focus of the project to the new cycle series

6
nova/api/openstack/wsgi_app.py
View File

@ -23,6 +23,7 @@ from nova import context
from nova import exception
from nova import objects
from nova import service
from nova import utils
CONF = cfg.CONF
@ -40,6 +41,11 @@ def _get_config_files(env=None):
def _setup_service(host, name):
try:
utils.raise_if_old_compute()
except exception.TooOldComputeService as e:
logging.getLogger(__name__).warning(str(e))
binary = name if name.startswith('nova-') else "nova-%s" % name
ctxt = context.get_admin_context()

8
nova/exception.py
View File

@ -559,6 +559,14 @@ class ServiceTooOld(Invalid):
"Unable to continue.")
class TooOldComputeService(Invalid):
msg_fmt = _("Current Nova version does not support computes older than "
"%(oldest_supported_version)s but the minimum compute service "
"level in your %(scope)s is %(min_service_level)d and the "
"oldest supported service level is "
"%(oldest_supported_service)d.")
class DestinationDiskExists(Invalid):
msg_fmt = _("The supplied disk path (%(path)s) already exists, "
"it is expected not to exist.")

7
nova/objects/service.py
View File

@ -192,6 +192,13 @@ SERVICE_VERSION_HISTORY = (
{'compute_rpc': '5.12'},
)
# This is used to raise an error at service startup if older than N-1 computes
# are detected. Update this at the beginning of every release cycle
OLDEST_SUPPORTED_SERVICE_VERSION = 'Ussuri'
SERVICE_VERSION_ALIASES = {
'Ussuri': 41
}
# TODO(berrange): Remove NovaObjectDictCompat
@base.NovaObjectRegistry.register

11
nova/service.py
View File

@ -255,6 +255,17 @@ class Service(service.Service):
periodic_fuzzy_delay=periodic_fuzzy_delay,
periodic_interval_max=periodic_interval_max)
# NOTE(gibi): This have to be after the service object creation as
# that is the point where we can safely use the RPC to the conductor.
# E.g. the Service.__init__ actually waits for the conductor to start
# up before it allows the service to be created. The
# raise_if_old_compute() depends on the RPC to be up and does not
# implement its own retry mechanism to connect to the conductor.
try:
utils.raise_if_old_compute()
except exception.TooOldComputeService as e:
LOG.warning(str(e))
return service_obj
def kill(self):

40
nova/tests/functional/test_service.py
View File

@ -10,7 +10,10 @@
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
from nova import context as nova_context
from nova.objects import service
from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.functional import fixtures as func_fixtures
@ -97,3 +100,40 @@ class ServiceTestCase(test.TestCase,
self.metadata.start()
# Cell cache should be empty after the service reset.
self.assertEqual({}, nova_context.CELL_CACHE)
class TestOldComputeCheck(
test.TestCase, integrated_helpers.InstanceHelperMixin):
def test_conductor_warns_if_old_compute(self):
old_version = service.SERVICE_VERSION_ALIASES[
service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
with mock.patch(
"nova.objects.service.get_minimum_version_all_cells",
return_value=old_version):
self.start_service('conductor')
self.assertIn(
'Current Nova version does not support computes older than',
self.stdlog.logger.output)
def test_api_warns_if_old_compute(self):
old_version = service.SERVICE_VERSION_ALIASES[
service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
with mock.patch(
"nova.objects.service.get_minimum_version_all_cells",
return_value=old_version):
self.useFixture(nova_fixtures.OSAPIFixture(api_version='v2.1'))
self.assertIn(
'Current Nova version does not support computes older than',
self.stdlog.logger.output)
def test_compute_warns_if_old_compute(self):
old_version = service.SERVICE_VERSION_ALIASES[
service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
with mock.patch(
"nova.objects.service.get_minimum_version_all_cells",
return_value=old_version):
self._start_compute('host1')
self.assertIn(
'Current Nova version does not support computes older than',
self.stdlog.logger.output)

3
nova/tests/unit/api/openstack/test_requestlog.py
View File

@ -42,7 +42,8 @@ class TestRequestLogMiddleware(testtools.TestCase):
# this is the minimal set of magic mocks needed to convince
# the API service it can start on it's own without a database.
mocks = ['nova.objects.Service.get_by_host_and_binary',
'nova.objects.Service.create']
'nova.objects.Service.create',
'nova.utils.raise_if_old_compute']
self.stdlog = fixtures.StandardLogging()
self.useFixture(self.stdlog)
for m in mocks:

1
nova/tests/unit/test_fixtures.py
View File

@ -90,6 +90,7 @@ class TestLogging(testtools.TestCase):
class TestOSAPIFixture(testtools.TestCase):
@mock.patch('nova.objects.Service.get_by_host_and_binary')
@mock.patch('nova.objects.Service.create')
@mock.patch('nova.utils.raise_if_old_compute', new=mock.Mock())
def test_responds_to_version(self, mock_service_create, mock_get):
"""Ensure the OSAPI server responds to calls sensibly."""
self.useFixture(output.CaptureOutput())

18
nova/tests/unit/test_service.py
View File

@ -268,6 +268,24 @@ class ServiceTestCase(test.NoDBTestCase):
serv.reset()
mock_reset.assert_called_once_with()
@mock.patch('nova.conductor.api.API.wait_until_ready')
@mock.patch('nova.utils.raise_if_old_compute')
def test_old_compute_version_check_happens_after_wait_for_conductor(
self, mock_check_old, mock_wait):
obj_base.NovaObject.indirection_api = mock.MagicMock()
def fake_wait(*args, **kwargs):
mock_check_old.assert_not_called()
mock_wait.side_effect = fake_wait
service.Service.create(
self.host, self.binary, self.topic,
'nova.tests.unit.test_service.FakeManager')
mock_check_old.assert_called_once_with()
mock_wait.assert_called_once_with(mock.ANY)
class TestWSGIService(test.NoDBTestCase):

87
nova/tests/unit/test_utils.py
View File

@ -37,6 +37,7 @@ from nova import context
from nova import exception
from nova.objects import base as obj_base
from nova.objects import instance as instance_obj
from nova.objects import service as service_obj
from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.unit.objects import test_objects
@ -1207,3 +1208,89 @@ class TestGetSDKAdapter(test.NoDBTestCase):
self.mock_get_confgrp.assert_called_once_with(self.service_type)
self.mock_connection.assert_not_called()
self.mock_get_auth_sess.assert_not_called()
class TestOldComputeCheck(test.NoDBTestCase):
@mock.patch('nova.objects.service.get_minimum_version_all_cells')
def test_no_compute(self, mock_get_min_service):
mock_get_min_service.return_value = 0
utils.raise_if_old_compute()
mock_get_min_service.assert_called_once_with(
mock.ANY, ['nova-compute'])
@mock.patch('nova.objects.service.get_minimum_version_all_cells')
def test_old_but_supported_compute(self, mock_get_min_service):
oldest = service_obj.SERVICE_VERSION_ALIASES[
service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
mock_get_min_service.return_value = oldest
utils.raise_if_old_compute()
mock_get_min_service.assert_called_once_with(
mock.ANY, ['nova-compute'])
@mock.patch('nova.objects.service.get_minimum_version_all_cells')
def test_new_compute(self, mock_get_min_service):
mock_get_min_service.return_value = service_obj.SERVICE_VERSION
utils.raise_if_old_compute()
mock_get_min_service.assert_called_once_with(
mock.ANY, ['nova-compute'])
@mock.patch('nova.objects.service.Service.get_minimum_version')
def test_too_old_compute_cell(self, mock_get_min_service):
self.flags(group='api_database', connection=None)
oldest = service_obj.SERVICE_VERSION_ALIASES[
service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
mock_get_min_service.return_value = oldest - 1
ex = self.assertRaises(
exception.TooOldComputeService, utils.raise_if_old_compute)
self.assertIn('cell', str(ex))
mock_get_min_service.assert_called_once_with(mock.ANY, 'nova-compute')
@mock.patch('nova.objects.service.get_minimum_version_all_cells')
def test_too_old_compute_top_level(self, mock_get_min_service):
self.flags(group='api_database', connection='fake db connection')
oldest = service_obj.SERVICE_VERSION_ALIASES[
service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
mock_get_min_service.return_value = oldest - 1
ex = self.assertRaises(
exception.TooOldComputeService, utils.raise_if_old_compute)
self.assertIn('system', str(ex))
mock_get_min_service.assert_called_once_with(
mock.ANY, ['nova-compute'])
@mock.patch.object(utils.LOG, 'warning')
@mock.patch('nova.objects.service.Service.get_minimum_version')
@mock.patch('nova.objects.service.get_minimum_version_all_cells')
def test_api_db_is_configured_but_the_service_cannot_access_db(
self, mock_get_all, mock_get, mock_warn):
# This is the case when the nova-compute service is wrongly configured
# with db credentials but nova-compute is never allowed to access the
# db directly.
mock_get_all.side_effect = exception.DBNotAllowed(
binary='nova-compute')
oldest = service_obj.SERVICE_VERSION_ALIASES[
service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
mock_get.return_value = oldest - 1
ex = self.assertRaises(
exception.TooOldComputeService, utils.raise_if_old_compute)
self.assertIn('cell', str(ex))
mock_get_all.assert_called_once_with(mock.ANY, ['nova-compute'])
mock_get.assert_called_once_with(mock.ANY, 'nova-compute')
mock_warn.assert_called_once_with(
'This service is configured for access to the API database but is '
'not allowed to directly access the database. You should run this '
'service without the [api_database]/connection config option. The '
'service version check will only query the local cell.')

50
nova/utils.py
View File

@ -1053,3 +1053,53 @@ def normalize_rc_name(rc_name):
norm_name = norm_name.upper()
norm_name = orc.CUSTOM_NAMESPACE + norm_name
return norm_name
def raise_if_old_compute():
# to avoid circular imports
from nova import context as nova_context
from nova.objects import service
ctxt = nova_context.get_admin_context()
if CONF.api_database.connection is not None:
scope = 'system'
try:
current_service_version = service.get_minimum_version_all_cells(
ctxt, ['nova-compute'])
except exception.DBNotAllowed:
# This most likely means we are in a nova-compute service
# which is configured with a connection to the API database.
# We should not be attempting to "get out" of our cell to
# look at the minimum versions of nova-compute services in other
# cells, so DBNotAllowed was raised. Leave a warning message
# and fall back to only querying computes in our cell.
LOG.warning(
'This service is configured for access to the API database '
'but is not allowed to directly access the database. You '
'should run this service without the '
'[api_database]/connection config option. The service version '
'check will only query the local cell.')
scope = 'cell'
current_service_version = service.Service.get_minimum_version(
ctxt, 'nova-compute')
else:
scope = 'cell'
# We in a cell so target our query to the current cell only
current_service_version = service.Service.get_minimum_version(
ctxt, 'nova-compute')
if current_service_version == 0:
# 0 means no compute in the system,
# probably a fresh install before the computes are registered
return
oldest_supported_service_level = service.SERVICE_VERSION_ALIASES[
service.OLDEST_SUPPORTED_SERVICE_VERSION]
if current_service_version < oldest_supported_service_level:
raise exception.TooOldComputeService(
oldest_supported_version=service.OLDEST_SUPPORTED_SERVICE_VERSION,
scope=scope,
min_service_level=current_service_version,
oldest_supported_service=oldest_supported_service_level)

9
releasenotes/notes/warn-when-services-started-with-old-compute-fc80b4ff58a2aaea.yaml Normal file
View File

@ -0,0 +1,9 @@
---
upgrade:
- |
Nova services only support old computes if the compute is not
older than the previous major nova release. From now on nova services will
emit a warning at startup if the deployment contains too old compute
services. From the 23.0.0 (Wallaby) release nova services will refuse to
start if the deployment contains too old compute services to prevent
compatibility issues.