From aa7c6f87699ec1340bd446a7d47e1453847a637f Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Mon, 29 Jun 2020 18:13:24 +0200 Subject: [PATCH] Prevent starting services with older than N-1 computes Nova services only support computes that are not older than the previous major release. This patch introduces a check in the service startup that prevents staring the service if too old computes are detected. Change-Id: Ie15ec8299ae52ae8f5334d591ed3944e9585cf71 --- doc/source/contributor/ptl-guide.rst | 3 + nova/api/openstack/wsgi_app.py | 3 + nova/exception.py | 8 +++ nova/objects/service.py | 7 +++ nova/service.py | 2 + nova/tests/functional/test_service.py | 45 +++++++++++++- .../unit/api/openstack/test_requestlog.py | 3 +- nova/tests/unit/test_fixtures.py | 1 + nova/tests/unit/test_service.py | 8 +++ nova/tests/unit/test_utils.py | 60 +++++++++++++++++++ nova/utils.py | 33 ++++++++++ ...ces-with-old-compute-fc80b4ff58a2aaea.yaml | 7 +++ 12 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/prevent-starting-services-with-old-compute-fc80b4ff58a2aaea.yaml diff --git a/doc/source/contributor/ptl-guide.rst b/doc/source/contributor/ptl-guide.rst index 3e4b2ab18c8a..6683ef5e942f 100644 --- a/doc/source/contributor/ptl-guide.rst +++ b/doc/source/contributor/ptl-guide.rst @@ -257,6 +257,9 @@ Immediately after RC * Example: https://review.opendev.org/543580 + * Bump the oldest supported compute service version + * https://review.opendev.org/#/c/738482/ + * Create the launchpad series for the next cycle * Set the development focus of the project to the new cycle series diff --git a/nova/api/openstack/wsgi_app.py b/nova/api/openstack/wsgi_app.py index c789da51bb52..7366df434ae3 100644 --- a/nova/api/openstack/wsgi_app.py +++ b/nova/api/openstack/wsgi_app.py @@ -23,6 +23,7 @@ from nova import context from nova import exception from nova import objects from nova import service +from nova import utils CONF = cfg.CONF @@ -40,6 +41,8 @@ def _get_config_files(env=None): def _setup_service(host, name): + utils.raise_if_old_compute() + binary = name if name.startswith('nova-') else "nova-%s" % name ctxt = context.get_admin_context() diff --git a/nova/exception.py b/nova/exception.py index f58705d8c1de..5585c29129d1 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -559,6 +559,14 @@ class ServiceTooOld(Invalid): "Unable to continue.") +class TooOldComputeService(Invalid): + msg_fmt = _("Current Nova version does not support computes older than " + "%(oldest_supported_version)s but the minimum compute service " + "level in your %(scope)s is %(min_service_level)d and the " + "oldest supported service level is " + "%(oldest_supported_service)d.") + + class DestinationDiskExists(Invalid): msg_fmt = _("The supplied disk path (%(path)s) already exists, " "it is expected not to exist.") diff --git a/nova/objects/service.py b/nova/objects/service.py index 33d1a5e2e8e0..5ce447f5368c 100644 --- a/nova/objects/service.py +++ b/nova/objects/service.py @@ -192,6 +192,13 @@ SERVICE_VERSION_HISTORY = ( {'compute_rpc': '5.12'}, ) +# This is used to raise an error at service startup if older than N-1 computes +# are detected. Update this at the beginning of every release cycle +OLDEST_SUPPORTED_SERVICE_VERSION = 'Victoria' +SERVICE_VERSION_ALIASES = { + 'Victoria': 52, +} + # TODO(berrange): Remove NovaObjectDictCompat @base.NovaObjectRegistry.register diff --git a/nova/service.py b/nova/service.py index 5af20ed0b9ad..5ce1c99cf246 100644 --- a/nova/service.py +++ b/nova/service.py @@ -249,6 +249,8 @@ class Service(service.Service): debugger.init() + utils.raise_if_old_compute() + service_obj = cls(host, binary, topic, manager, report_interval=report_interval, periodic_enable=periodic_enable, diff --git a/nova/tests/functional/test_service.py b/nova/tests/functional/test_service.py index 962245cf7077..5e6a649d4092 100644 --- a/nova/tests/functional/test_service.py +++ b/nova/tests/functional/test_service.py @@ -10,7 +10,11 @@ # License for the specific language governing permissions and limitations # under the License. +from unittest import mock + from nova import context as nova_context +from nova import exception +from nova.objects import service from nova import test from nova.tests import fixtures as nova_fixtures from nova.tests.functional import fixtures as func_fixtures @@ -93,7 +97,44 @@ class ServiceTestCase(test.TestCase, self._wait_for_state_change(server, 'ACTIVE') # Cell cache should be populated after creating a server. self.assertTrue(nova_context.CELL_CACHE) - self.metadata.stop() - self.metadata.start() + # we need to mock nova.utils.raise_if_old_compute() that is run at + # service startup as that will check the global service level which + # populates the cell cache + with mock.patch("nova.utils.raise_if_old_compute"): + self.metadata.stop() + self.metadata.start() # Cell cache should be empty after the service reset. self.assertEqual({}, nova_context.CELL_CACHE) + + +class TestOldComputeCheck( + test.TestCase, integrated_helpers.InstanceHelperMixin): + + def test_conductor_fails_to_start_with_old_compute(self): + old_version = service.SERVICE_VERSION_ALIASES[ + service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1 + with mock.patch( + "nova.objects.service.get_minimum_version_all_cells", + return_value=old_version): + self.assertRaises( + exception.TooOldComputeService, self.start_service, + 'conductor') + + def test_api_fails_to_start_with_old_compute(self): + old_version = service.SERVICE_VERSION_ALIASES[ + service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1 + with mock.patch( + "nova.objects.service.get_minimum_version_all_cells", + return_value=old_version): + self.assertRaises( + exception.TooOldComputeService, self.useFixture, + nova_fixtures.OSAPIFixture(api_version='v2.1')) + + def test_compute_fails_to_start_with_old_compute(self): + old_version = service.SERVICE_VERSION_ALIASES[ + service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1 + with mock.patch( + "nova.objects.service.get_minimum_version_all_cells", + return_value=old_version): + self.assertRaises( + exception.TooOldComputeService, self._start_compute, 'host1') diff --git a/nova/tests/unit/api/openstack/test_requestlog.py b/nova/tests/unit/api/openstack/test_requestlog.py index 386b79a52893..0a25f7c51f5f 100644 --- a/nova/tests/unit/api/openstack/test_requestlog.py +++ b/nova/tests/unit/api/openstack/test_requestlog.py @@ -42,7 +42,8 @@ class TestRequestLogMiddleware(testtools.TestCase): # this is the minimal set of magic mocks needed to convince # the API service it can start on it's own without a database. mocks = ['nova.objects.Service.get_by_host_and_binary', - 'nova.objects.Service.create'] + 'nova.objects.Service.create', + 'nova.utils.raise_if_old_compute'] self.stdlog = fixtures.StandardLogging() self.useFixture(self.stdlog) for m in mocks: diff --git a/nova/tests/unit/test_fixtures.py b/nova/tests/unit/test_fixtures.py index fbc7a3f6a13e..797a4ef63030 100644 --- a/nova/tests/unit/test_fixtures.py +++ b/nova/tests/unit/test_fixtures.py @@ -90,6 +90,7 @@ class TestLogging(testtools.TestCase): class TestOSAPIFixture(testtools.TestCase): @mock.patch('nova.objects.Service.get_by_host_and_binary') @mock.patch('nova.objects.Service.create') + @mock.patch('nova.utils.raise_if_old_compute', new=mock.Mock()) def test_responds_to_version(self, mock_service_create, mock_get): """Ensure the OSAPI server responds to calls sensibly.""" self.useFixture(output.CaptureOutput()) diff --git a/nova/tests/unit/test_service.py b/nova/tests/unit/test_service.py index 8e0ed56ae0b2..7464c56d1a99 100644 --- a/nova/tests/unit/test_service.py +++ b/nova/tests/unit/test_service.py @@ -268,6 +268,14 @@ class ServiceTestCase(test.NoDBTestCase): serv.reset() mock_reset.assert_called_once_with() + @mock.patch('nova.utils.raise_if_old_compute') + def test_old_compute_version_is_checked(self, mock_check_old): + service.Service.create( + self.host, self.binary, self.topic, + 'nova.tests.unit.test_service.FakeManager') + + mock_check_old.assert_called_once_with() + class TestWSGIService(test.NoDBTestCase): diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index e9930fcb7aa7..885aeb83d2dd 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -37,6 +37,7 @@ from nova import context from nova import exception from nova.objects import base as obj_base from nova.objects import instance as instance_obj +from nova.objects import service as service_obj from nova import test from nova.tests import fixtures as nova_fixtures from nova.tests.unit.objects import test_objects @@ -1207,3 +1208,62 @@ class TestGetSDKAdapter(test.NoDBTestCase): self.mock_get_confgrp.assert_called_once_with(self.service_type) self.mock_connection.assert_not_called() self.mock_get_auth_sess.assert_not_called() + + +class TestOldComputeCheck(test.NoDBTestCase): + + @mock.patch('nova.objects.service.get_minimum_version_all_cells') + def test_no_compute(self, mock_get_min_service): + mock_get_min_service.return_value = 0 + + utils.raise_if_old_compute() + + mock_get_min_service.assert_called_once_with( + mock.ANY, ['nova-compute']) + + @mock.patch('nova.objects.service.get_minimum_version_all_cells') + def test_old_but_supported_compute(self, mock_get_min_service): + oldest = service_obj.SERVICE_VERSION_ALIASES[ + service_obj.OLDEST_SUPPORTED_SERVICE_VERSION] + mock_get_min_service.return_value = oldest + + utils.raise_if_old_compute() + + mock_get_min_service.assert_called_once_with( + mock.ANY, ['nova-compute']) + + @mock.patch('nova.objects.service.get_minimum_version_all_cells') + def test_new_compute(self, mock_get_min_service): + mock_get_min_service.return_value = service_obj.SERVICE_VERSION + + utils.raise_if_old_compute() + + mock_get_min_service.assert_called_once_with( + mock.ANY, ['nova-compute']) + + @mock.patch('nova.objects.service.Service.get_minimum_version') + def test_too_old_compute_cell(self, mock_get_min_service): + self.flags(group='api_database', connection=None) + oldest = service_obj.SERVICE_VERSION_ALIASES[ + service_obj.OLDEST_SUPPORTED_SERVICE_VERSION] + mock_get_min_service.return_value = oldest - 1 + + ex = self.assertRaises( + exception.TooOldComputeService, utils.raise_if_old_compute) + + self.assertIn('cell', str(ex)) + mock_get_min_service.assert_called_once_with(mock.ANY, 'nova-compute') + + @mock.patch('nova.objects.service.get_minimum_version_all_cells') + def test_too_old_compute_top_level(self, mock_get_min_service): + self.flags(group='api_database', connection='fake db connection') + oldest = service_obj.SERVICE_VERSION_ALIASES[ + service_obj.OLDEST_SUPPORTED_SERVICE_VERSION] + mock_get_min_service.return_value = oldest - 1 + + ex = self.assertRaises( + exception.TooOldComputeService, utils.raise_if_old_compute) + + self.assertIn('system', str(ex)) + mock_get_min_service.assert_called_once_with( + mock.ANY, ['nova-compute']) diff --git a/nova/utils.py b/nova/utils.py index 0a40fa6ffc26..014d2b632e6f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -1053,3 +1053,36 @@ def normalize_rc_name(rc_name): norm_name = norm_name.upper() norm_name = orc.CUSTOM_NAMESPACE + norm_name return norm_name + + +def raise_if_old_compute(): + # to avoid circular imports + from nova import context as nova_context + from nova.objects import service + + ctxt = nova_context.get_admin_context() + + if CONF.api_database.connection is not None: + scope = 'system' + current_service_version = service.get_minimum_version_all_cells( + ctxt, ['nova-compute']) + else: + scope = 'cell' + # We in a cell so target our query to the current cell only + current_service_version = service.Service.get_minimum_version( + ctxt, 'nova-compute') + + if current_service_version == 0: + # 0 means no compute in the system, + # probably a fresh install before the computes are registered + return + + oldest_supported_service_level = service.SERVICE_VERSION_ALIASES[ + service.OLDEST_SUPPORTED_SERVICE_VERSION] + + if current_service_version < oldest_supported_service_level: + raise exception.TooOldComputeService( + oldest_supported_version=service.OLDEST_SUPPORTED_SERVICE_VERSION, + scope=scope, + min_service_level=current_service_version, + oldest_supported_service=oldest_supported_service_level) diff --git a/releasenotes/notes/prevent-starting-services-with-old-compute-fc80b4ff58a2aaea.yaml b/releasenotes/notes/prevent-starting-services-with-old-compute-fc80b4ff58a2aaea.yaml new file mode 100644 index 000000000000..8d061d764a26 --- /dev/null +++ b/releasenotes/notes/prevent-starting-services-with-old-compute-fc80b4ff58a2aaea.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Nova services only support old computes if the compute is not + older than the previous major nova release. To prevent compatibility + issues at run time nova services will refuse to start if the deployment + contains too old compute services.