Add cross-cell resize policy rule and enable in API

This adds the "compute:servers:resize:cross_cell" policy
rule which is now used in the API to determine if a resize
or cold migrate operation can be performed across cells.

The check in the API is based on:

- The policy check passing for the request.
- The minimum nova-compute service version being high
  enough across all cells to perform a cross-cell resize.

If either of those conditions fail a traditional same-cell
resize will be performed.

A docs stub is added here and will be fleshed out in an
upcoming patch.

Implements blueprint cross-cell-resize

Change-Id: Ie8a0f79a3b16e02b7a34a1b81f547013a3d88996

doc/source/admin/configuration/cross-cell-resize.rst

@@ -0,0 +1,26 @@
+=================
+Cross-cell resize
+=================
+
+Train spec: https://specs.openstack.org/openstack/nova-specs/specs/train/approved/cross-cell-resize.html
+
+.. todo:: Flesh this out to describe what cross-cell resize is, how it is
+  triggered (policy), related configuration (long_rpc_timeout) including
+  the CrossCellWeigher, limitations and known issues, recovering from failures
+  during a cross-cell resize, maybe a flow chart for the overall process in
+  the code, minimum upgrade requirements and supported drivers (libvirt-only
+  at this time).
+
+Limitations
+~~~~~~~~~~~
+
+These are known to not yet be supported in the code:
+
+* Instances with ports attached that have bandwidth-aware resource provider
+  allocations.
+* Reschedules
+
+These are likely not to work since they have not been validated by testing:
+
+* Instances with PCI devices attached.
+* Instances with a NUMA topology.

doc/source/admin/configuration/index.rst

@@ -21,6 +21,7 @@ A list of config options based on different topics can be found below:
 
    /admin/configuration/api
    /admin/configuration/resize
+   /admin/configuration/cross-cell-resize
    /admin/configuration/fibre-channel
    /admin/configuration/iscsi-offload
    /admin/configuration/hypervisors

doc/source/admin/configuration/resize.rst

@@ -6,6 +6,8 @@ Resize (or Server resize) is the ability to change the flavor of a server, thus
 allowing it to upscale or downscale according to user needs.  For this feature
 to work properly, you might need to configure some underlying virt layers.
 
+For cross-cell resize, refer to :doc:`/admin/configuration/cross-cell-resize`.
+
 Virt drivers
 ------------
 

nova/compute/api.py

@@ -105,6 +105,7 @@ AGGREGATE_ACTION_DELETE = 'Delete'
 AGGREGATE_ACTION_ADD = 'Add'
 
 MIN_COMPUTE_SYNC_COMPUTE_STATUS_DISABLED = 38
+MIN_COMPUTE_CROSS_CELL_RESIZE = 47
 
 # FIXME(danms): Keep a global cache of the cells we find the
 # first time we look. This needs to be refreshed on a timer or
@@ -3724,20 +3725,30 @@ class API(base.Base):
         :param instance: Instance object being resized
         :returns: True if cross-cell resize is allowed, False otherwise
         """
-        # TODO(mriedem): Uncomment this when confirm/revert are done. For now
+        # First check to see if the requesting project/user is allowed by
-        # this method can just be used to internally enable the feature for
+        # policy to perform cross-cell resize.
-        # functional testing.
+        allowed = context.can(
-
+            servers_policies.CROSS_CELL_RESIZE,
-        # TODO(mriedem): If true, we should probably check if all of the
+            target={'user_id': instance.user_id,
-        # computes are running a high enough service version and if not, do
+                    'project_id': instance.project_id},
-        # what? Raise an exception or just log something and return False?
+            fatal=False)
-
+        # If the user is allowed by policy, check to make sure the deployment
-        # return context.can(
+        # is upgraded to the point of supporting cross-cell resize on all
-        #     server_policies.CROSS_CELL_RESIZE,
+        # compute services.
-        #     target={'user_id': instance.user_id,
+        if allowed:
-        #             'project_id': instance.project_id},
+            # TODO(mriedem): We can remove this minimum compute version check
-        #     fatal=False)
+            # in the 21.0.0 "U" release.
-        return False
+            min_compute_version = (
+                objects.service.get_minimum_version_all_cells(
+                    context, ['nova-compute']))
+            if min_compute_version < MIN_COMPUTE_CROSS_CELL_RESIZE:
+                LOG.debug('Request is allowed by policy to perform cross-cell '
+                          'resize but the minimum nova-compute service '
+                          'version in the deployment %s is less than %s so '
+                          'cross-cell resize is not allowed at this time.',
+                          min_compute_version, MIN_COMPUTE_CROSS_CELL_RESIZE)
+                allowed = False
+        return allowed
 
     @staticmethod
     def _validate_host_for_cold_migrate(

nova/policies/base.py

@@ -12,9 +12,10 @@
 
 from oslo_policy import policy
 
-RULE_ADMIN_OR_OWNER = 'rule:admin_or_owner'
+RULE_ADMIN_OR_OWNER = 'rule:admin_or_owner'  # Admins or owners of the resource
-RULE_ADMIN_API = 'rule:admin_api'
+RULE_ADMIN_API = 'rule:admin_api'  # Allow only users with the admin role
-RULE_ANY = '@'
+RULE_ANY = '@'  # Any user is allowed to perform the action.
+RULE_NOBODY = '!'  # No users are allowed to perform the action.
 
 # TODO(gmann): # Special string ``system_scope:all`` is added for system
 # scoped policies for backwards compatibility where ``nova.conf [oslo_policy]

nova/policies/servers.py

@@ -21,6 +21,7 @@ SERVERS = 'os_compute_api:servers:%s'
 NETWORK_ATTACH_EXTERNAL = 'network:attach_external_network'
 ZERO_DISK_FLAVOR = SERVERS % 'create:zero_disk_flavor'
 REQUESTED_DESTINATION = 'compute:servers:create:requested_destination'
+CROSS_CELL_RESIZE = 'compute:servers:resize:cross_cell'
 
 rules = [
     policy.DocumentedRuleDefault(
@@ -319,6 +320,19 @@ https://bugs.launchpad.net/nova/+bug/1739646 for details.
                 'path': '/servers/{server_id}/action (resize)'
             }
         ]),
+    policy.DocumentedRuleDefault(
+        CROSS_CELL_RESIZE,
+        base.RULE_NOBODY,
+        "Resize a server across cells. By default, this is disabled for all "
+        "users and recommended to be tested in a deployment for admin users "
+        "before opening it up to non-admin users. Resizing within a cell is "
+        "the default preferred behavior even if this is enabled. ",
+        [
+            {
+                'method': 'POST',
+                'path': '/servers/{server_id}/action (resize)'
+            }
+        ]),
     policy.DocumentedRuleDefault(
         SERVERS % 'rebuild',
         RULE_AOO,

nova/tests/functional/test_cross_cell_migrate.py

@@ -20,6 +20,8 @@ from nova import context as nova_context
 from nova.db import api as db_api
 from nova import exception
 from nova import objects
+from nova.policies import base as base_policies
+from nova.policies import servers as servers_policies
 from nova.scheduler import utils as scheduler_utils
 from nova.scheduler import weights
 from nova.tests import fixtures as nova_fixtures
@@ -90,15 +92,10 @@ class TestMultiCellMigrate(integrated_helpers.ProviderUsageBaseTestCase):
         # Enable cross-cell resize policy since it defaults to not allow
         # anyone to perform that type of operation. For these tests we'll
         # just allow admins to perform cross-cell resize.
-        # TODO(mriedem): Uncomment this when the policy rule is added and
+        self.policy.set_rules({
-        # used in the compute API _allow_cross_cell_resize method. For now
+            servers_policies.CROSS_CELL_RESIZE:
-        # we just stub that method to return True.
+                base_policies.RULE_ADMIN_API},
-        # self.policy_fixture.set_rules({
+            overwrite=False)
-        #     servers_policies.CROSS_CELL_RESIZE:
-        #         base_policies.RULE_ADMIN_API},
-        #     overwrite=False)
-        self.stub_out('nova.compute.api.API._allow_cross_cell_resize',
-                      lambda *a, **kw: True)
 
     def assertFlavorMatchesAllocation(self, flavor, allocation,
                                       volume_backed=False):

nova/tests/unit/compute/test_compute_api.py

@@ -6987,6 +6987,49 @@ class ComputeAPIUnitTestCase(_ComputeAPIUnitTestMixIn, test.NoDBTestCase):
         mock_conductor_confirm.assert_called_once_with(
             self.context, instance, migration)
 
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_allow_cross_cell_resize_default_false(self, mock_get_min_ver):
+        """Based on the default policy this asserts nobody is allowed to
+        perform cross-cell resize.
+        """
+        instance = objects.Instance(
+            project_id='fake-project', user_id='fake-user')
+        self.assertFalse(self.compute_api._allow_cross_cell_resize(
+            self.context, instance))
+        # We did not need to check the minimum nova-compute version since the
+        # policy check failed.
+        mock_get_min_ver.assert_not_called()
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells',
+                return_value=compute_api.MIN_COMPUTE_CROSS_CELL_RESIZE - 1)
+    def test_allow_cross_cell_resize_false_old_version(self, mock_get_min_ver):
+        """Policy allows cross-cell resize but minimum nova-compute service
+        version is not new enough.
+        """
+        instance = objects.Instance(
+            project_id='fake-project', user_id='fake-user')
+        with mock.patch.object(self.context, 'can', return_value=True) as can:
+            self.assertFalse(self.compute_api._allow_cross_cell_resize(
+                self.context, instance))
+        can.assert_called_once()
+        mock_get_min_ver.assert_called_once_with(
+            self.context, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells',
+                return_value=compute_api.MIN_COMPUTE_CROSS_CELL_RESIZE)
+    def test_allow_cross_cell_resize_true(self, mock_get_min_ver):
+        """Policy allows cross-cell resize and minimum nova-compute service
+        version is new enough.
+        """
+        instance = objects.Instance(
+            project_id='fake-project', user_id='fake-user')
+        with mock.patch.object(self.context, 'can', return_value=True) as can:
+            self.assertTrue(self.compute_api._allow_cross_cell_resize(
+                self.context, instance))
+        can.assert_called_once()
+        mock_get_min_ver.assert_called_once_with(
+            self.context, ['nova-compute'])
+
 
 class DiffDictTestCase(test.NoDBTestCase):
     """Unit tests for _diff_dict()."""

nova/tests/unit/test_policy.py

@@ -24,6 +24,7 @@ import requests_mock
 
 from nova import context
 from nova import exception
+from nova.policies import servers as servers_policy
 from nova import policy
 from nova import test
 from nova.tests.unit import fake_policy
@@ -460,6 +461,10 @@ class RealRolePolicyTestCase(test.NoDBTestCase):
 "os_compute_api:os-services:list",
 )
 
+        self.allow_nobody_rules = (
+            servers_policy.CROSS_CELL_RESIZE,
+        )
+
     def test_all_rules_in_sample_file(self):
         special_rules = ["context_is_admin", "admin_or_owner", "default"]
         for (name, rule) in self.fake_policy.items():
@@ -485,6 +490,12 @@ class RealRolePolicyTestCase(test.NoDBTestCase):
         for rule in self.allow_all_rules:
             policy.authorize(self.non_admin_context, rule, self.target)
 
+    def test_allow_nobody_rules(self):
+        """No one can perform these operations, not even admin."""
+        for rule in self.allow_nobody_rules:
+            self.assertRaises(exception.PolicyNotAuthorized, policy.authorize,
+                              self.admin_context, rule, self.target)
+
     def test_rule_missing(self):
         rules = policy.get_rules()
         # eliqiao os_compute_api:os-quota-class-sets:show requires
@@ -497,5 +508,6 @@ class RealRolePolicyTestCase(test.NoDBTestCase):
                          'system_admin_or_owner', 'system_or_project_reader')
         result = set(rules.keys()) - set(self.admin_only_rules +
             self.admin_or_owner_rules +
-            self.allow_all_rules + self.system_reader_rules + special_rules)
+            self.allow_all_rules + self.system_reader_rules +
+            self.allow_nobody_rules + special_rules)
         self.assertEqual(set([]), result)

releasenotes/notes/cross-cell-resize-37a735adadbafe91.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Cross-cell resize is now supported but is disabled by default for all
+    users. Refer to the `administrator documentation`__ for details.
+
+    .. __: https://docs.openstack.org/nova/latest/admin/configuration/cross-cell-resize.html