Add nova-status upgrade check for consoles

This will check if a deployment is currently using consoles and warns
the operator to set [workarounds]enable_consoleauth = True on their
console proxy host if they are performing a rolling upgrade which is
not yet complete.

Partial-Bug: #1798188

Change-Id: Idd6079ce4038d6f19966e98bcc61422b61b3636b

doc/source/cli/nova-status.rst

@@ -121,6 +121,9 @@ Upgrade
   **19.0.0 (Stein)**
 
   * Checks for the Placement API are modified to require version 1.30.
+  * Checks are added for the **nova-consoleauth** service to warn and provide
+    additional instructions to set **[workarounds]enable_consoleauth = True**
+    while performing a live/rolling upgrade.
 
 See Also
 ========

nova/cmd/status.py

@@ -575,6 +575,87 @@ class UpgradeCommands(object):
             return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
         return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
 
+    def _check_console_auths(self):
+        """Checks for console usage and warns with info for rolling upgrade.
+
+        Iterates all cells checking to see if the nova-consoleauth service is
+        non-deleted/non-disabled and whether there are any console token auths
+        in that cell database. If there is a nova-consoleauth service being
+        used and no console token auths in the cell database, emit a warning
+        telling the user to set [workarounds]enable_consoleauth = True if they
+        are performing a rolling upgrade.
+        """
+        # If we're using cells v1, we don't need to check if the workaround
+        # needs to be used because cells v1 always uses nova-consoleauth.
+        # If the operator has already enabled the workaround, we don't need
+        # to check anything.
+        if CONF.cells.enable or CONF.workarounds.enable_consoleauth:
+            return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
+
+        # We need to check cell0 for nova-consoleauth service records because
+        # it's possible a deployment could have services stored in the cell0
+        # database, if they've defaulted their [database]connection in
+        # nova.conf to cell0.
+        meta = MetaData(bind=db_session.get_api_engine())
+        cell_mappings = Table('cell_mappings', meta, autoload=True)
+        mappings = cell_mappings.select().execute().fetchall()
+
+        if not mappings:
+            # There are no cell mappings so we can't determine this, just
+            # return a warning. The cellsv2 check would have already failed
+            # on this.
+            msg = (_('Unable to check consoles without cell mappings.'))
+            return UpgradeCheckResult(UpgradeCheckCode.WARNING, msg)
+
+        ctxt = nova_context.get_admin_context()
+        # If we find a non-deleted, non-disabled nova-consoleauth service in
+        # any cell, we will assume the deployment is using consoles.
+        using_consoles = False
+        for mapping in mappings:
+            with nova_context.target_cell(ctxt, mapping) as cctxt:
+                # Check for any non-deleted, non-disabled nova-consoleauth
+                # service.
+                meta = MetaData(bind=db_session.get_engine(context=cctxt))
+                services = Table('services', meta, autoload=True)
+                consoleauth_service_record = (
+                    select([services.c.id]).select_from(services).where(and_(
+                        services.c.binary == 'nova-consoleauth',
+                        services.c.deleted == 0,
+                        services.c.disabled == false())).execute().first())
+                if consoleauth_service_record:
+                    using_consoles = True
+                    break
+
+        if using_consoles:
+            # If the deployment is using consoles, we can only be certain the
+            # upgrade is complete if each compute service is >= Rocky and
+            # supports storing console token auths in the database backend.
+            for mapping in mappings:
+                # Skip cell0 as no compute services should be in it.
+                if mapping['uuid'] == cell_mapping_obj.CellMapping.CELL0_UUID:
+                    continue
+                # Get the minimum nova-compute service version in this
+                # cell.
+                with nova_context.target_cell(ctxt, mapping) as cctxt:
+                    min_version = self._get_min_service_version(
+                        cctxt, 'nova-compute')
+                    # We could get None for the minimum version in the case of
+                    # new install where there are no computes. If there are
+                    # compute services, they should all have versions.
+                    if min_version is not None and min_version < 35:
+                        msg = _("One or more cells were found which have "
+                                "nova-compute services older than Rocky. "
+                                "Please set the "
+                                "'[workarounds]enable_consoleauth' "
+                                "configuration option to 'True' on your "
+                                "console proxy host if you are performing a "
+                                "rolling upgrade to enable consoles to "
+                                "function during a partial upgrade.")
+                        return UpgradeCheckResult(UpgradeCheckCode.WARNING,
+                                                  msg)
+
+        return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
+
     # The format of the check functions is to return an UpgradeCheckResult
     # object with the appropriate UpgradeCheckCode and details set. If the
     # check hits warnings or failures then those should be stored in the
@@ -595,6 +676,8 @@ class UpgradeCommands(object):
         (_('API Service Version'), _check_api_service_version),
         # Added in Rocky
         (_('Request Spec Migration'), _check_request_spec_migration),
+        # Added in Stein (but also useful going back to Rocky)
+        (_('Console Auths'), _check_console_auths),
     )
 
     def _get_details(self, upgrade_check_result):

nova/tests/unit/cmd/test_status.py

@@ -1040,3 +1040,177 @@ class TestUpgradeCheckRequestSpecMigration(test.NoDBTestCase):
                       "'nova-manage db online_data_migrations' on each cell "
                       "to create the missing request specs." %
                       self.cell_mappings['cell2'].uuid, result.details)
+
+
+class TestUpgradeCheckConsoles(test.NoDBTestCase):
+    """Tests for the nova-status upgrade check for consoles."""
+
+    # We'll setup the database ourselves because we need to use cells fixtures
+    # for multiple cell mappings.
+    USES_DB_SELF = True
+
+    # This will create three cell mappings: cell0, cell1 (default) and cell2
+    NUMBER_OF_CELLS = 2
+
+    def setUp(self):
+        super(TestUpgradeCheckConsoles, self).setUp()
+        self.output = StringIO()
+        self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
+        # We always need the API DB to be setup.
+        self.useFixture(nova_fixtures.Database(database='api'))
+        self.cmd = status.UpgradeCommands()
+
+    @staticmethod
+    def _create_service_in_cell(ctxt, cell, binary, is_deleted=False,
+                                disabled=False, version=None,
+                                create_token_auth=False):
+        with context.target_cell(ctxt, cell) as cctxt:
+            service = objects.Service(context=cctxt, binary=binary,
+                                      disabled=disabled, host='dontcare')
+            if version:
+                service.version = version
+            service.create()
+
+            if is_deleted:
+                service.destroy()
+
+            if create_token_auth:
+                # We have to create an instance in order to create a token
+                # auth.
+                inst = objects.Instance(context=cctxt,
+                                        uuid=uuidutils.generate_uuid())
+                inst.create()
+                auth = objects.ConsoleAuthToken(context=cctxt,
+                                                console_type='novnc',
+                                                host='hostname', port=6080,
+                                                instance_uuid=inst.uuid)
+                auth.authorize(CONF.consoleauth.token_ttl)
+
+        return service
+
+    def test_check_cells_v1_enabled(self):
+        """This is a 'success' case since the console auths check is
+        ignored when running cells v1.
+        """
+        self.flags(enable=True, group='cells')
+        result = self.cmd._check_console_auths()
+        self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
+
+    def test_check_workaround_enabled(self):
+        """This is a 'success' case since the console auths check is
+        ignored when the workaround is already enabled.
+        """
+        self.flags(enable_consoleauth=True, group='workarounds')
+        result = self.cmd._check_console_auths()
+        self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
+
+    def test_deleted_disabled_consoleauth(self):
+        """Tests that services other than nova-consoleauth and deleted/disabled
+        nova-consoleauth services are filtered out.
+        """
+        self._setup_cells()
+        ctxt = context.get_admin_context()
+
+        # Create a compute service in cell1.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-compute')
+        # Create a deleted consoleauth service in cell1.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-consoleauth', is_deleted=True)
+        # Create a compute service in cell2.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
+                                     'nova-compute')
+        # Create a disabled consoleauth service in cell2.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
+                                     'nova-consoleauth', disabled=True)
+
+        result = self.cmd._check_console_auths()
+        self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
+
+    def test_consoleauth_with_upgrade_not_started(self):
+        """Tests the scenario where the deployment is using consoles but has no
+        compute services >= Rocky, i.e. a not started upgrade.
+        """
+        self._setup_cells()
+        ctxt = context.get_admin_context()
+
+        # Create a deleted consoleauth service in cell1.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-consoleauth', is_deleted=True)
+        # Create a live consoleauth service in cell0. (Asserts we check cell0).
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell0'],
+                                     'nova-consoleauth')
+        # Create Queens compute services in the cells.
+        for cell in ['cell1', 'cell2']:
+            self._create_service_in_cell(ctxt, self.cell_mappings[cell],
+                                         'nova-compute', version=30)
+
+        result = self.cmd._check_console_auths()
+        self.assertEqual(status.UpgradeCheckCode.WARNING, result.code)
+
+    def test_consoleauth_with_upgrade_complete(self):
+        """Tests the scenario where the deployment is using consoles and has
+        all compute services >= Rocky in every cell database, i.e. a completed
+        upgrade.
+        """
+        self._setup_cells()
+        ctxt = context.get_admin_context()
+
+        # Create a live consoleauth service in cell1 with token auth.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-consoleauth',
+                                     create_token_auth=True)
+
+        # Create a live consoleauth service in cell2 with token auth.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
+                                     'nova-consoleauth',
+                                     create_token_auth=True)
+
+        # Create Rocky compute services in the cells.
+        for cell in ['cell1', 'cell2']:
+            self._create_service_in_cell(ctxt, self.cell_mappings[cell],
+                                         'nova-compute', version=35)
+
+        # Create a Queens compute service in cell0. This not actually valid,
+        # we do it to assert that we skip cell0 when checking service versions.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell0'],
+                                     'nova-compute', version=30)
+
+        result = self.cmd._check_console_auths()
+        self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code)
+
+    def test_consoleauth_with_upgrade_partial(self):
+        """Tests the scenario where the deployment is using consoles and has
+        compute services >= Rocky in at least one, but not all, cell databases,
+        i.e. a partial upgrade.
+        """
+        self._setup_cells()
+        ctxt = context.get_admin_context()
+
+        # Create a live consoleauth service in cell1.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-consoleauth')
+
+        # Create a live consoleauth service in cell2 with token auth.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
+                                     'nova-consoleauth',
+                                     create_token_auth=True)
+
+        # Create a Queens compute service in cell1.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell1'],
+                                     'nova-compute', version=30)
+
+        # Create a Rocky compute service in cell2.
+        self._create_service_in_cell(ctxt, self.cell_mappings['cell2'],
+                                     'nova-compute', version=35)
+
+        result = self.cmd._check_console_auths()
+
+        self.assertEqual(status.UpgradeCheckCode.WARNING, result.code)
+        self.assertIn("One or more cells were found which have nova-compute "
+                      "services older than Rocky. "
+                      "Please set the '[workarounds]enable_consoleauth' "
+                      "configuration option to 'True' on your console proxy "
+                      "host if you are performing a rolling upgrade to enable "
+                      "consoles to function during a partial upgrade.",
+                      result.details)

releasenotes/notes/nova-status-check-consoleauths-618acb3a67f97418.yaml

@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    A new check is added to the ``nova-status upgrade check`` CLI to check for
+    use of the ``nova-consoleauth`` service to warn and provide additional
+    instructions to set ``[workarounds]enable_consoleauth = True`` while
+    performing a live/rolling upgrade.