Add support for promoting a failed over backend

Change-Id: Ib9c34b0806b71e2088ac0fa9886ad8abd7a2a45f
Implements: blueprint cheesecake-promote-backend

cinder/cmd/manage.py

@@ -410,6 +410,27 @@ class DbCommands(object):
 
         sys.exit(1 if ran else 0)
 
+    @args('--enable-replication', action='store_true', default=False,
+          help='Set replication status to enabled (default: %(default)s).')
+    @args('--active-backend-id', default=None,
+          help='Change the active backend ID (default: %(default)s).')
+    @args('--backend-host', required=True,
+          help='The backend host name.')
+    def reset_active_backend(self, enable_replication, active_backend_id,
+                             backend_host):
+        """Reset the active backend for a host."""
+
+        ctxt = context.get_admin_context()
+
+        try:
+            db.reset_active_backend(ctxt, enable_replication,
+                                    active_backend_id, backend_host)
+        except db_exc.DBReferenceError:
+            print(_("Failed to reset active backend for host %s, "
+                    "check cinder-manage logs for more details.") %
+                  backend_host)
+            sys.exit(1)
+
 
 class VersionCommands(object):
     """Class for exposing the codebase version."""

cinder/db/api.py

@@ -1609,6 +1609,16 @@ def get_booleans_for_table(table_name):
 ###################
 
 
+def reset_active_backend(context, enable_replication, active_backend_id,
+                         backend_host):
+    """Reset the active backend for a host."""
+    return IMPL.reset_active_backend(context, enable_replication,
+                                     active_backend_id, backend_host)
+
+
+###################
+
+
 def driver_initiator_data_insert_by_key(context, initiator,
                                         namespace, key, value):
     """Updates DriverInitiatorData entry.

cinder/db/sqlalchemy/api.py

@@ -58,6 +58,7 @@ from cinder import db
 from cinder.db.sqlalchemy import models
 from cinder import exception
 from cinder.i18n import _
+from cinder import objects
 from cinder.objects import fields
 from cinder import utils
 from cinder.volume import utils as vol_utils
@@ -6490,6 +6491,41 @@ def purge_deleted_rows(context, age_in_days):
 ###############################
 
 
+@require_admin_context
+def reset_active_backend(context, enable_replication, active_backend_id,
+                         backend_host):
+
+    service = objects.Service.get_by_host_and_topic(context,
+                                                    backend_host,
+                                                    'cinder-volume',
+                                                    disabled=True)
+    if not service.frozen:
+        raise exception.ServiceUnavailable(
+            'Service for host %(host)s must first be frozen.' %
+            {'host': backend_host})
+
+    actions = {
+        'disabled': False,
+        'disabled_reason': '',
+        'active_backend_id': None,
+        'replication_status': 'enabled',
+    }
+
+    expectations = {
+        'frozen': True,
+        'disabled': True,
+    }
+
+    if service.is_clustered:
+        service.cluster.conditional_update(actions, expectations)
+        service.cluster.reset_service_replication()
+    else:
+        service.conditional_update(actions, expectations)
+
+
+###############################
+
+
 def _translate_messages(messages):
     return [_translate_message(message) for message in messages]
 

cinder/objects/cluster.py

@@ -169,6 +169,24 @@ class Cluster(base.CinderPersistentObject, base.CinderObject,
         return (self.last_heartbeat and
                 self.last_heartbeat >= utils.service_expired_time(True))
 
+    def reset_service_replication(self):
+        """Reset service replication flags on promotion.
+
+        When an admin promotes a cluster, each service member requires an
+        update to maintain database consistency.
+        """
+        actions = {
+            'replication_status': 'enabled',
+            'active_backend_id': None,
+        }
+
+        expectations = {
+            'cluster_name': self.name,
+        }
+
+        db.conditional_update(self._context, objects.Service.model,
+                              actions, expectations)
+
 
 @base.CinderObjectRegistry.register
 class ClusterList(base.ObjectListBase, base.CinderObject):

cinder/tests/unit/db/test_reset_backend.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests for resetting active backend replication parameters."""
+
+from cinder import db
+from cinder import exception
+from cinder.tests.unit import test_db_api
+from cinder.tests.unit import utils
+
+
+class ResetActiveBackendCase(test_db_api.BaseTest):
+    """Unit tests for cinder.db.api.reset_active_backend."""
+
+    def test_enabled_service(self):
+        """Test that enabled services cannot be queried."""
+        service_overrides = {'topic': 'cinder-volume'}
+        service = utils.create_service(self.ctxt, values=service_overrides)
+        self.assertRaises(exception.ServiceNotFound,
+                          db.reset_active_backend,
+                          self.ctxt, True, 'fake-backend-id',
+                          service.host)
+
+    def test_disabled_service(self):
+        """Test that non-frozen services are rejected."""
+        service_overrides = {'topic': 'cinder-volume',
+                             'disabled': True}
+        service = utils.create_service(self.ctxt, values=service_overrides)
+        self.assertRaises(exception.ServiceUnavailable,
+                          db.reset_active_backend,
+                          self.ctxt, True, 'fake-backend-id',
+                          service.host)
+
+    def test_disabled_and_frozen_service(self):
+        """Test that disabled and frozen services are updated correctly."""
+        service_overrides = {'topic': 'cinder-volume',
+                             'disabled': True,
+                             'frozen': True,
+                             'replication_status': 'failed-over',
+                             'active_backend_id': 'seconary'}
+        service = utils.create_service(self.ctxt, values=service_overrides)
+        db.reset_active_backend(self.ctxt, True, 'fake-backend-id',
+                                service.host)
+        db_service = db.service_get(self.ctxt, service.id)
+
+        self.assertFalse(db_service.disabled)
+        self.assertEqual('', db_service.disabled_reason)
+        self.assertIsNone(db_service.active_backend_id)
+        self.assertEqual('enabled', db_service.replication_status)

cinder/tests/unit/objects/test_cluster.py

@@ -17,6 +17,8 @@ import ddt
 import mock
 from oslo_utils import timeutils
 
+import cinder.db
+from cinder.db.sqlalchemy import models
 from cinder import objects
 from cinder.tests.unit import fake_cluster
 from cinder.tests.unit import objects as test_objects
@@ -112,6 +114,15 @@ class TestCluster(test_objects.BaseObjectsTestCase):
                                                 last_heartbeat=expired_time)
         self.assertFalse(cluster.is_up)
 
+    @mock.patch.object(cinder.db, 'conditional_update')
+    def test_reset_service_replication(self, mock_update):
+        cluster = fake_cluster.fake_cluster_ovo(self.context)
+        cluster.reset_service_replication()
+        mock_update.assert_called_with(self.context, models.Service,
+                                       {'replication_status': 'enabled',
+                                        'active_backend_id': None},
+                                       {'cluster_name': cluster.name})
+
     @ddt.data('1.0', '1.1')
     def tests_obj_make_compatible(self, version):
         new_fields = {'replication_status': 'error', 'frozen': True,

cinder/tests/unit/test_cmd.py

@@ -477,6 +477,16 @@ class TestCinderManageCmd(test.TestCase):
         self.assertEqual(127, exit.code)
         cinder_manage.DbCommands.online_migrations[0].assert_not_called()
 
+    @mock.patch('cinder.db.reset_active_backend')
+    @mock.patch('cinder.context.get_admin_context')
+    def test_db_commands_reset_active_backend(self, admin_ctxt_mock,
+                                              reset_backend_mock):
+        db_cmds = cinder_manage.DbCommands()
+        db_cmds.reset_active_backend(True, 'fake-backend-id', 'fake-host')
+        reset_backend_mock.assert_called_with(admin_ctxt_mock.return_value,
+                                              True, 'fake-backend-id',
+                                              'fake-host')
+
     @mock.patch('cinder.version.version_string')
     def test_versions_commands_list(self, version_string):
         version_cmds = cinder_manage.VersionCommands()

releasenotes/notes/cheesecake-promotion-30a3336fb911c3ad.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    A new cinder-manage command, reset_active_backend, was added to promote a
+    failed-over backend participating in replication.  This allows you to
+    reset a backend without manually editing the database.  A backend
+    undergoing promotion using this command is expected to be in a disabled
+    and frozen state.  Support for both standalone and clustered backend
+    configurations are supported.