Prevent starting services with older than N-1 computes

Nova services only support computes that are not older than
the previous major release. This patch introduces a check in the
service startup that prevents staring the service if too old computes
are detected.

Change-Id: Ie15ec8299ae52ae8f5334d591ed3944e9585cf71

doc/source/contributor/ptl-guide.rst

@@ -257,6 +257,9 @@ Immediately after RC
 
     * Example: https://review.opendev.org/543580
 
+  * Bump the oldest supported compute service version
+    * https://review.opendev.org/#/c/738482/
+
 * Create the launchpad series for the next cycle
 
 * Set the development focus of the project to the new cycle series

nova/api/openstack/wsgi_app.py

@@ -23,6 +23,7 @@ from nova import context
 from nova import exception
 from nova import objects
 from nova import service
+from nova import utils
 
 CONF = cfg.CONF
 
@@ -40,6 +41,8 @@ def _get_config_files(env=None):
 
 
 def _setup_service(host, name):
+    utils.raise_if_old_compute()
+
     binary = name if name.startswith('nova-') else "nova-%s" % name
 
     ctxt = context.get_admin_context()

nova/exception.py

@@ -559,6 +559,14 @@ class ServiceTooOld(Invalid):
                 "Unable to continue.")
 
 
+class TooOldComputeService(Invalid):
+    msg_fmt = _("Current Nova version does not support computes older than "
+                "%(oldest_supported_version)s but the minimum compute service "
+                "level in your %(scope)s is %(min_service_level)d and the "
+                "oldest supported service level is "
+                "%(oldest_supported_service)d.")
+
+
 class DestinationDiskExists(Invalid):
     msg_fmt = _("The supplied disk path (%(path)s) already exists, "
                 "it is expected not to exist.")

nova/objects/service.py

@@ -192,6 +192,13 @@ SERVICE_VERSION_HISTORY = (
     {'compute_rpc': '5.12'},
 )
 
+# This is used to raise an error at service startup if older than N-1 computes
+# are detected. Update this at the beginning of every release cycle
+OLDEST_SUPPORTED_SERVICE_VERSION = 'Victoria'
+SERVICE_VERSION_ALIASES = {
+    'Victoria': 52,
+}
+
 
 # TODO(berrange): Remove NovaObjectDictCompat
 @base.NovaObjectRegistry.register

nova/service.py

@@ -249,6 +249,8 @@ class Service(service.Service):
 
         debugger.init()
 
+        utils.raise_if_old_compute()
+
         service_obj = cls(host, binary, topic, manager,
                           report_interval=report_interval,
                           periodic_enable=periodic_enable,

nova/tests/functional/test_service.py

@@ -10,7 +10,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from unittest import mock
+
 from nova import context as nova_context
+from nova import exception
+from nova.objects import service
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional import fixtures as func_fixtures
@@ -93,7 +97,44 @@ class ServiceTestCase(test.TestCase,
         self._wait_for_state_change(server, 'ACTIVE')
         # Cell cache should be populated after creating a server.
         self.assertTrue(nova_context.CELL_CACHE)
-        self.metadata.stop()
-        self.metadata.start()
+        # we need to mock nova.utils.raise_if_old_compute() that is run at
+        # service startup as that will check the global service level which
+        # populates the cell cache
+        with mock.patch("nova.utils.raise_if_old_compute"):
+            self.metadata.stop()
+            self.metadata.start()
         # Cell cache should be empty after the service reset.
         self.assertEqual({}, nova_context.CELL_CACHE)
+
+
+class TestOldComputeCheck(
+        test.TestCase, integrated_helpers.InstanceHelperMixin):
+
+    def test_conductor_fails_to_start_with_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self.assertRaises(
+                exception.TooOldComputeService, self.start_service,
+                'conductor')
+
+    def test_api_fails_to_start_with_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self.assertRaises(
+                exception.TooOldComputeService, self.useFixture,
+                nova_fixtures.OSAPIFixture(api_version='v2.1'))
+
+    def test_compute_fails_to_start_with_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self.assertRaises(
+                exception.TooOldComputeService, self._start_compute, 'host1')

nova/tests/unit/api/openstack/test_requestlog.py

@@ -42,7 +42,8 @@ class TestRequestLogMiddleware(testtools.TestCase):
         # this is the minimal set of magic mocks needed to convince
         # the API service it can start on it's own without a database.
         mocks = ['nova.objects.Service.get_by_host_and_binary',
-                 'nova.objects.Service.create']
+                 'nova.objects.Service.create',
+                 'nova.utils.raise_if_old_compute']
         self.stdlog = fixtures.StandardLogging()
         self.useFixture(self.stdlog)
         for m in mocks:

nova/tests/unit/test_fixtures.py

@@ -90,6 +90,7 @@ class TestLogging(testtools.TestCase):
 class TestOSAPIFixture(testtools.TestCase):
     @mock.patch('nova.objects.Service.get_by_host_and_binary')
     @mock.patch('nova.objects.Service.create')
+    @mock.patch('nova.utils.raise_if_old_compute', new=mock.Mock())
     def test_responds_to_version(self, mock_service_create, mock_get):
         """Ensure the OSAPI server responds to calls sensibly."""
         self.useFixture(output.CaptureOutput())

nova/tests/unit/test_service.py

@@ -268,6 +268,14 @@ class ServiceTestCase(test.NoDBTestCase):
             serv.reset()
             mock_reset.assert_called_once_with()
 
+    @mock.patch('nova.utils.raise_if_old_compute')
+    def test_old_compute_version_is_checked(self, mock_check_old):
+        service.Service.create(
+            self.host, self.binary, self.topic,
+            'nova.tests.unit.test_service.FakeManager')
+
+        mock_check_old.assert_called_once_with()
+
 
 class TestWSGIService(test.NoDBTestCase):
 

nova/tests/unit/test_utils.py

@@ -37,6 +37,7 @@ from nova import context
 from nova import exception
 from nova.objects import base as obj_base
 from nova.objects import instance as instance_obj
+from nova.objects import service as service_obj
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.unit.objects import test_objects
@@ -1207,3 +1208,62 @@ class TestGetSDKAdapter(test.NoDBTestCase):
         self.mock_get_confgrp.assert_called_once_with(self.service_type)
         self.mock_connection.assert_not_called()
         self.mock_get_auth_sess.assert_not_called()
+
+
+class TestOldComputeCheck(test.NoDBTestCase):
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_no_compute(self, mock_get_min_service):
+        mock_get_min_service.return_value = 0
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_old_but_supported_compute(self, mock_get_min_service):
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_new_compute(self, mock_get_min_service):
+        mock_get_min_service.return_value = service_obj.SERVICE_VERSION
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.Service.get_minimum_version')
+    def test_too_old_compute_cell(self, mock_get_min_service):
+        self.flags(group='api_database', connection=None)
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest - 1
+
+        ex = self.assertRaises(
+            exception.TooOldComputeService, utils.raise_if_old_compute)
+
+        self.assertIn('cell', str(ex))
+        mock_get_min_service.assert_called_once_with(mock.ANY, 'nova-compute')
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_too_old_compute_top_level(self, mock_get_min_service):
+        self.flags(group='api_database', connection='fake db connection')
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest - 1
+
+        ex = self.assertRaises(
+            exception.TooOldComputeService, utils.raise_if_old_compute)
+
+        self.assertIn('system', str(ex))
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])

nova/utils.py

@@ -1053,3 +1053,36 @@ def normalize_rc_name(rc_name):
     norm_name = norm_name.upper()
     norm_name = orc.CUSTOM_NAMESPACE + norm_name
     return norm_name
+
+
+def raise_if_old_compute():
+    # to avoid circular imports
+    from nova import context as nova_context
+    from nova.objects import service
+
+    ctxt = nova_context.get_admin_context()
+
+    if CONF.api_database.connection is not None:
+        scope = 'system'
+        current_service_version = service.get_minimum_version_all_cells(
+            ctxt, ['nova-compute'])
+    else:
+        scope = 'cell'
+        # We in a cell so target our query to the current cell only
+        current_service_version = service.Service.get_minimum_version(
+            ctxt, 'nova-compute')
+
+    if current_service_version == 0:
+        # 0 means no compute in the system,
+        # probably a fresh install before the computes are registered
+        return
+
+    oldest_supported_service_level = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION]
+
+    if current_service_version < oldest_supported_service_level:
+        raise exception.TooOldComputeService(
+            oldest_supported_version=service.OLDEST_SUPPORTED_SERVICE_VERSION,
+            scope=scope,
+            min_service_level=current_service_version,
+            oldest_supported_service=oldest_supported_service_level)

releasenotes/notes/prevent-starting-services-with-old-compute-fc80b4ff58a2aaea.yaml

@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    Nova services only support old computes if the compute is not
+    older than the previous major nova release. To prevent compatibility
+    issues at run time nova services will refuse to start if the deployment
+    contains too old compute services.