Merge "api: add soft-affinity policies for server groups"

doc/api_samples/versions/v21-version-get-resp.json

@@ -19,7 +19,7 @@
             }
         ],
         "status": "CURRENT",
-        "version": "2.14",
+        "version": "2.15",
         "min_version": "2.1",
         "updated": "2013-07-23T11:33:21Z"
     }

doc/api_samples/versions/versions-get-resp.json

@@ -22,7 +22,7 @@
                 }
             ],
             "status": "CURRENT",
-            "version": "2.14",
+            "version": "2.15",
             "min_version": "2.1",
             "updated": "2013-07-23T11:33:21Z"
         }

nova/api/openstack/api_version_request.py

@@ -56,6 +56,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
     * 2.13 - Add project id and user id information for os-server-groups API
     * 2.14 - Remove onSharedStorage from evacuate request body and remove
              adminPass from the response body
+    * 2.15 - Add soft-affinity and soft-anti-affinity policies
 """
 
 # The minimum and maximum versions of the API supported
@@ -64,7 +65,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
 # Note(cyeoh): This only applies for the v2.1 API once microversions
 # support is fully merged. It does not affect the V2 API.
 _MIN_API_VERSION = "2.1"
-_MAX_API_VERSION = "2.14"
+_MAX_API_VERSION = "2.15"
 DEFAULT_API_VERSION = _MIN_API_VERSION
 
 

nova/api/openstack/compute/schemas/server_groups.py

@@ -11,15 +11,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import copy
 
 from nova.api.validation import parameter_types
 
 # NOTE(russellb) There is one other policy, 'legacy', but we don't allow that
 # being set via the API.  It's only used when a group gets automatically
 # created to support the legacy behavior of the 'group' scheduler hint.
-SUPPORTED_POLICIES = ['anti-affinity', 'affinity']
-
-
 create = {
     'type': 'object',
     'properties': {
@@ -29,7 +27,7 @@ create = {
                 'name': parameter_types.name,
                 'policies': {
                     'type': 'array',
-                    'items': [{'enum': SUPPORTED_POLICIES}],
+                    'items': [{'enum': ['anti-affinity', 'affinity']}],
                     'uniqueItems': True,
                     'additionalItems': False,
                 }
@@ -41,3 +39,7 @@ create = {
     'required': ['server_group'],
     'additionalProperties': False,
 }
+
+create_v215 = copy.deepcopy(create)
+policies = create_v215['properties']['server_group']['properties']['policies']
+policies['items'][0]['enum'].extend(['soft-anti-affinity', 'soft-affinity'])

nova/api/openstack/compute/server_groups.py

@@ -131,8 +131,10 @@ class ServerGroupController(wsgi.Controller):
                   for group in limited_list]
         return {'server_groups': result}
 
+    @wsgi.Controller.api_version("2.1")
     @extensions.expected_errors((400, 403))
-    @validation.schema(schema.create)
+    @validation.schema(schema.create, "2.1", "2.14")
+    @validation.schema(schema.create_v215, "2.15")
     def create(self, req, body):
         """Creates a new server group."""
         context = _authorize_context(req)

nova/api/openstack/rest_api_version_history.rst

@@ -144,3 +144,9 @@ user documentation.
   automatically detect if the instance is on shared storage.
   Also adminPass is removed from the response body. The user can get the
   password with the server's os-server-password action.
+
+2.15
+----
+
+  From this version of the API users can choose 'soft-affinity' and
+  'soft-anti-affinity' rules too for server-groups.

nova/tests/functional/api/client.py

@@ -197,13 +197,16 @@ class TestOpenStackClient(object):
         kwargs.setdefault('check_response_status', [200])
         return APIResponse(self.api_request(relative_uri, **kwargs))
 
-    def api_post(self, relative_uri, body, **kwargs):
+    def api_post(self, relative_uri, body, api_version=None, **kwargs):
         kwargs['method'] = 'POST'
         if body:
             headers = kwargs.setdefault('headers', {})
             headers['Content-Type'] = 'application/json'
             kwargs['body'] = jsonutils.dumps(body)
 
+        if api_version:
+            headers['X-OpenStack-Nova-API-Version'] = api_version
+
         kwargs.setdefault('check_response_status', [200, 202])
         return APIResponse(self.api_request(relative_uri, **kwargs))
 
@@ -346,8 +349,9 @@ class TestOpenStackClient(object):
         return self.api_get('/os-server-groups/%s' %
                             group_id).body['server_group']
 
-    def post_server_groups(self, group):
+    def post_server_groups(self, group, api_version=None):
-        response = self.api_post('/os-server-groups', {"server_group": group})
+        response = self.api_post('/os-server-groups', {"server_group": group},
+                                 api_version)
         return response.body['server_group']
 
     def delete_server_group(self, group_id):

nova/tests/functional/api_sample_tests/api_samples/versions/v21-version-get-resp.json.tpl

@@ -19,7 +19,7 @@
             }
         ],
         "status": "CURRENT",
-        "version": "2.14",
+        "version": "2.15",
         "min_version": "2.1",
         "updated": "2013-07-23T11:33:21Z"
     }

nova/tests/functional/api_sample_tests/api_samples/versions/versions-get-resp.json.tpl

@@ -22,7 +22,7 @@
                 }
             ],
             "status": "CURRENT",
-            "version": "2.14",
+            "version": "2.15",
             "min_version": "2.1",
             "updated": "2013-07-23T11:33:21Z"
         }

nova/tests/functional/test_server_group.py

@@ -21,6 +21,7 @@ from oslo_config import cfg
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional.api import client
+from nova.tests.functional import api_paste_fixture
 from nova.tests.unit import fake_network
 from nova.tests.unit import policy_fixture
 
@@ -34,6 +35,7 @@ CONF = cfg.CONF
 class ServerGroupTestBase(test.TestCase):
     REQUIRES_LOCKING = True
     api_major_version = 'v2'
+    microversion = None
     _image_ref_parameter = 'imageRef'
     _flavor_ref_parameter = 'flavorRef'
 
@@ -50,14 +52,25 @@ class ServerGroupTestBase(test.TestCase):
     anti_affinity = {'name': 'fake-name-1', 'policies': ['anti-affinity']}
     affinity = {'name': 'fake-name-2', 'policies': ['affinity']}
 
+    def _get_weight_classes(self):
+        return []
+
     def setUp(self):
         super(ServerGroupTestBase, self).setUp()
         self.flags(scheduler_default_filters=self._scheduler_default_filters)
+        self.flags(scheduler_weight_classes=self._get_weight_classes())
         self.flags(service_down_time=self._service_down_time)
         self.flags(report_interval=self._report_interval)
 
         self.useFixture(policy_fixture.RealPolicyFixture())
-        api_fixture = self.useFixture(nova_fixtures.OSAPIFixture())
+
+        if self.api_major_version == 'v2.1':
+            api_fixture = self.useFixture(nova_fixtures.OSAPIFixture(
+                api_version='v2.1'))
+        else:
+            self.useFixture(api_paste_fixture.ApiPasteLegacyV2Fixture())
+            api_fixture = self.useFixture(nova_fixtures.OSAPIFixture(
+                api_version='v2'))
 
         self.api = api_fixture.api
         self.admin_api = api_fixture.admin_api
@@ -123,29 +136,13 @@ class ServerGroupTestBase(test.TestCase):
         server['name'] = name
         return server
 
-
+    def _test_create_delete_groups(self, groups):
-class ServerGroupTest(ServerGroupTestBase):
-
-    def setUp(self):
-        super(ServerGroupTest, self).setUp()
-
-        self.start_service('network')
-        self.compute = self.start_service('compute')
-
-        # NOTE(gibi): start a second compute host to be able to test affinity
-        self.compute2 = self.start_service('compute', host='host2')
-        fake_network.set_stub_network_methods(self.stubs)
-
-    def test_get_no_groups(self):
-        groups = self.api.get_server_groups()
-        self.assertEqual([], groups)
-
-    def test_create_and_delete_groups(self):
-        groups = [self.anti_affinity,
-                  self.affinity]
         created_groups = []
         for group in groups:
-            created_group = self.api.post_server_groups(group)
+            created_group = self.api.post_server_groups(
+                group, api_version=self.microversion)
+            created_group.pop('user_id', None)
+            created_group.pop('project_id', None)
             created_groups.append(created_group)
             self.assertEqual(group['name'], created_group['name'])
             self.assertEqual(group['policies'], created_group['policies'])
@@ -167,11 +164,37 @@ class ServerGroupTest(ServerGroupTestBase):
             existing_groups = self.api.get_server_groups()
             self.assertNotIn(group, existing_groups)
 
+
+class ServerGroupTestV2(ServerGroupTestBase):
+    api_major_version = 'v2'
+
+    def setUp(self):
+        super(ServerGroupTestV2, self).setUp()
+
+        self.start_service('network')
+        self.compute = self.start_service('compute')
+
+        # NOTE(gibi): start a second compute host to be able to test affinity
+        self.compute2 = self.start_service('compute', host='host2')
+        self.addCleanup(self.compute.kill)
+        self.addCleanup(self.compute2.kill)
+        fake_network.set_stub_network_methods(self.stubs)
+
+    def test_get_no_groups(self):
+        groups = self.api.get_server_groups()
+        self.assertEqual([], groups)
+
+    def test_create_and_delete_groups(self):
+        groups = [self.anti_affinity,
+                  self.affinity]
+        self._test_create_delete_groups(groups)
+
     def test_create_wrong_policy(self):
         ex = self.assertRaises(client.OpenStackApiException,
                                self.api.post_server_groups,
                                {'name': 'fake-name-1',
-                                'policies': ['wrong-policy']})
+                                'policies': ['wrong-policy']},
+                               api_version=self.microversion)
         self.assertEqual(400, ex.response.status_code)
         self.assertIn('Invalid input', ex.response.text)
         self.assertIn('wrong-policy', ex.response.text)
@@ -207,6 +230,21 @@ class ServerGroupTest(ServerGroupTestBase):
         self.assertNotIn(openstack_group, all_projects_non_admin)
         self.assertIn(openstack1_group, all_projects_non_admin)
 
+    def test_create_duplicated_policy(self):
+        ex = self.assertRaises(client.OpenStackApiException,
+                               self.api.post_server_groups,
+                               {"name": "fake-name-1",
+                                "policies": ["affinity", "affinity"]})
+        self.assertEqual(400, ex.response.status_code)
+        self.assertIn('Invalid input', ex.response.text)
+
+    def test_create_multiple_policies(self):
+        ex = self.assertRaises(client.OpenStackApiException,
+                               self.api.post_server_groups,
+                               {"name": "fake-name-1",
+                                "policies": ["anti-affinity", "affinity"]})
+        self.assertEqual(400, ex.response.status_code)
+
     def _boot_servers_to_group(self, group, flavor=None):
         servers = []
         for _ in range(0, 2):
@@ -216,7 +254,8 @@ class ServerGroupTest(ServerGroupTestBase):
         return servers
 
     def test_boot_servers_with_affinity(self):
-        created_group = self.api.post_server_groups(self.affinity)
+        created_group = self.api.post_server_groups(
+            self.affinity, api_version=self.microversion)
         servers = self._boot_servers_to_group(created_group)
 
         members = self.api.get_server_group(created_group['id'])['members']
@@ -226,7 +265,8 @@ class ServerGroupTest(ServerGroupTestBase):
             self.assertEqual(host, server['OS-EXT-SRV-ATTR:host'])
 
     def test_boot_servers_with_affinity_no_valid_host(self):
-        created_group = self.api.post_server_groups(self.affinity)
+        created_group = self.api.post_server_groups(
+            self.affinity, api_version=self.microversion)
         # Using big enough flavor to use up the resources on the host
         flavor = self.api.get_flavors()[2]
         self._boot_servers_to_group(created_group, flavor=flavor)
@@ -262,7 +302,8 @@ class ServerGroupTest(ServerGroupTestBase):
                          failed_server['fault']['message'])
 
     def _rebuild_with_group(self, group):
-        created_group = self.api.post_server_groups(group)
+        created_group = self.api.post_server_groups(
+            group, api_version=self.microversion)
         servers = self._boot_servers_to_group(created_group)
 
         post = {'rebuild': {self._image_ref_parameter:
@@ -407,6 +448,7 @@ class ServerGroupTest(ServerGroupTestBase):
 
 
 class ServerGroupAffinityConfTest(ServerGroupTestBase):
+    api_major_version = 'v2.1'
     # Load only anti-affinity filter so affinity will be missing
     _scheduler_default_filters = 'ServerGroupAntiAffinityFilter'
 
@@ -423,6 +465,7 @@ class ServerGroupAffinityConfTest(ServerGroupTestBase):
 
 
 class ServerGroupAntiAffinityConfTest(ServerGroupTestBase):
+    api_major_version = 'v2.1'
     # Load only affinity filter so anti-affinity will be missing
     _scheduler_default_filters = 'ServerGroupAffinityFilter'
 
@@ -438,5 +481,233 @@ class ServerGroupAntiAffinityConfTest(ServerGroupTestBase):
         self.assertEqual(400, failed_server['fault']['code'])
 
 
-class ServerGroupTestV21(ServerGroupTest):
+class ServerGroupSoftAffinityConfTest(ServerGroupTestBase):
     api_major_version = 'v2.1'
+    microversion = '2.15'
+    soft_affinity = {'name': 'fake-name-4',
+                     'policies': ['soft-affinity']}
+
+    def _get_weight_classes(self):
+        # Load only soft-anti-affinity weigher so affinity will be missing
+        return ['nova.scheduler.weights.affinity.'
+                'ServerGroupSoftAntiAffinityWeigher']
+
+    @mock.patch('nova.scheduler.utils._SUPPORTS_SOFT_AFFINITY', None)
+    def test_soft_affinity_no_filter(self):
+        created_group = self.api.post_server_groups(self.soft_affinity,
+                                                    self.microversion)
+
+        failed_server = self._boot_a_server_to_group(created_group,
+                                                     expected_status='ERROR')
+        self.assertEqual('ServerGroup policy is not supported: '
+                         'ServerGroupSoftAffinityWeigher not configured',
+                         failed_server['fault']['message'])
+        self.assertEqual(400, failed_server['fault']['code'])
+
+
+class ServerGroupSoftAntiAffinityConfTest(ServerGroupTestBase):
+    api_major_version = 'v2.1'
+    microversion = '2.15'
+    soft_anti_affinity = {'name': 'fake-name-3',
+                          'policies': ['soft-anti-affinity']}
+
+    # Load only soft affinity filter so anti-affinity will be missing
+    _scheduler_weight_classes = ['nova.scheduler.weights.affinity.'
+                                 'ServerGroupSoftAffinityWeigher']
+
+    def _get_weight_classes(self):
+        # Load only soft affinity filter so anti-affinity will be missing
+        return ['nova.scheduler.weights.affinity.'
+                'ServerGroupSoftAffinityWeigher']
+
+    @mock.patch('nova.scheduler.utils._SUPPORTS_SOFT_ANTI_AFFINITY', None)
+    def test_soft_anti_affinity_no_filter(self):
+        created_group = self.api.post_server_groups(
+            self.soft_anti_affinity, api_version=self.microversion)
+
+        failed_server = self._boot_a_server_to_group(created_group,
+                                                     expected_status='ERROR')
+        self.assertEqual('ServerGroup policy is not supported: '
+                         'ServerGroupSoftAntiAffinityWeigher not configured',
+                         failed_server['fault']['message'])
+        self.assertEqual(400, failed_server['fault']['code'])
+
+
+class ServerGroupTestV21(ServerGroupTestV2):
+    api_major_version = 'v2.1'
+
+    def test_soft_affinity_not_supported(self):
+        ex = self.assertRaises(client.OpenStackApiException,
+                               self.api.post_server_groups,
+                               {'name': 'fake-name-1',
+                                'policies': ['soft-affinity']})
+        self.assertEqual(400, ex.response.status_code)
+        self.assertIn('Invalid input', ex.response.text)
+        self.assertIn('soft-affinity', ex.response.text)
+
+
+class ServerGroupTestV215(ServerGroupTestV2):
+    api_major_version = 'v2.1'
+    microversion = '2.15'
+
+    soft_anti_affinity = {'name': 'fake-name-3',
+                          'policies': ['soft-anti-affinity']}
+    soft_affinity = {'name': 'fake-name-4',
+                     'policies': ['soft-affinity']}
+
+    def setUp(self):
+        super(ServerGroupTestV215, self).setUp()
+
+        soft_affinity_patcher = mock.patch(
+            'nova.scheduler.utils._SUPPORTS_SOFT_AFFINITY')
+        soft_anti_affinity_patcher = mock.patch(
+            'nova.scheduler.utils._SUPPORTS_SOFT_ANTI_AFFINITY')
+        self.addCleanup(soft_affinity_patcher.stop)
+        self.addCleanup(soft_anti_affinity_patcher.stop)
+        self.mock_soft_affinity = soft_affinity_patcher.start()
+        self.mock_soft_anti_affinity = soft_anti_affinity_patcher.start()
+        self.mock_soft_affinity.return_value = None
+        self.mock_soft_anti_affinity.return_value = None
+
+    def _get_weight_classes(self):
+        return ['nova.scheduler.weights.affinity.'
+                'ServerGroupSoftAffinityWeigher',
+                'nova.scheduler.weights.affinity.'
+                'ServerGroupSoftAntiAffinityWeigher']
+
+    def test_create_and_delete_groups(self):
+        groups = [self.anti_affinity,
+                  self.affinity,
+                  self.soft_affinity,
+                  self.soft_anti_affinity]
+        self._test_create_delete_groups(groups)
+
+    def test_boot_servers_with_soft_affinity(self):
+        created_group = self.api.post_server_groups(
+            self.soft_affinity, api_version=self.microversion)
+        servers = self._boot_servers_to_group(created_group)
+        members = self.api.get_server_group(created_group['id'])['members']
+
+        self.assertEqual(2, len(servers))
+        self.assertIn(servers[0]['id'], members)
+        self.assertIn(servers[1]['id'], members)
+        self.assertEqual(servers[0]['OS-EXT-SRV-ATTR:host'],
+                         servers[1]['OS-EXT-SRV-ATTR:host'])
+
+    def test_boot_servers_with_soft_affinity_no_resource_on_first_host(self):
+        created_group = self.api.post_server_groups(
+            self.soft_affinity, api_version=self.microversion)
+
+        # Using big enough flavor to use up the resources on the first host
+        flavor = self.api.get_flavors()[2]
+        servers = self._boot_servers_to_group(created_group, flavor)
+
+        # The third server cannot be booted on the first host as there
+        # is not enough resource there, but as opposed to the affinity policy
+        # it will be booted on the other host, which has enough resources.
+        third_server = self._boot_a_server_to_group(created_group,
+                                                    flavor=flavor)
+        members = self.api.get_server_group(created_group['id'])['members']
+        hosts = []
+        for server in servers:
+            hosts.append(server['OS-EXT-SRV-ATTR:host'])
+
+        self.assertIn(third_server['id'], members)
+        self.assertNotIn(third_server['OS-EXT-SRV-ATTR:host'], hosts)
+
+    def test_boot_servers_with_soft_anti_affinity(self):
+        created_group = self.api.post_server_groups(
+            self.soft_anti_affinity, api_version=self.microversion)
+        servers = self._boot_servers_to_group(created_group)
+        members = self.api.get_server_group(created_group['id'])['members']
+
+        self.assertEqual(2, len(servers))
+        self.assertIn(servers[0]['id'], members)
+        self.assertIn(servers[1]['id'], members)
+        self.assertNotEqual(servers[0]['OS-EXT-SRV-ATTR:host'],
+                            servers[1]['OS-EXT-SRV-ATTR:host'])
+
+    def test_boot_servers_with_soft_anti_affinity_one_available_host(self):
+        self.compute2.kill()
+        created_group = self.api.post_server_groups(
+            self.soft_anti_affinity, api_version=self.microversion)
+        servers = self._boot_servers_to_group(created_group)
+
+        members = self.api.get_server_group(created_group['id'])['members']
+        host = servers[0]['OS-EXT-SRV-ATTR:host']
+        for server in servers:
+            self.assertIn(server['id'], members)
+            self.assertEqual(host, server['OS-EXT-SRV-ATTR:host'])
+
+    def test_rebuild_with_soft_affinity(self):
+        untouched_server, rebuilt_server = self._rebuild_with_group(
+            self.soft_affinity)
+        self.assertEqual(untouched_server['OS-EXT-SRV-ATTR:host'],
+                         rebuilt_server['OS-EXT-SRV-ATTR:host'])
+
+    def test_rebuild_with_soft_anti_affinity(self):
+        untouched_server, rebuilt_server = self._rebuild_with_group(
+            self.soft_anti_affinity)
+        self.assertNotEqual(untouched_server['OS-EXT-SRV-ATTR:host'],
+                            rebuilt_server['OS-EXT-SRV-ATTR:host'])
+
+    def _migrate_with_soft_affinity_policies(self, group):
+        created_group = self.api.post_server_groups(
+            group, api_version=self.microversion)
+        servers = self._boot_servers_to_group(created_group)
+
+        post = {'migrate': {}}
+        self.admin_api.post_server_action(servers[1]['id'], post)
+        migrated_server = self._wait_for_state_change(servers[1],
+                                                      'VERIFY_RESIZE')
+
+        return [migrated_server['OS-EXT-SRV-ATTR:host'],
+                servers[0]['OS-EXT-SRV-ATTR:host']]
+
+    def test_migrate_with_soft_affinity(self):
+        migrated_server, other_server = (
+            self._migrate_with_soft_affinity_policies(self.soft_affinity))
+        self.assertNotEqual(migrated_server, other_server)
+
+    def test_migrate_with_soft_anti_affinity(self):
+        migrated_server, other_server = (
+            self._migrate_with_soft_affinity_policies(self.soft_anti_affinity))
+        self.assertEqual(migrated_server, other_server)
+
+    def _evacuate_with_soft_anti_affinity_policies(self, group):
+        created_group = self.api.post_server_groups(
+            group, api_version=self.microversion)
+        servers = self._boot_servers_to_group(created_group)
+
+        host = self._get_compute_service_by_host_name(
+            servers[1]['OS-EXT-SRV-ATTR:host'])
+        host.stop()
+        # Need to wait service_down_time amount of seconds to ensure
+        # nova considers the host down
+        time.sleep(self._service_down_time)
+
+        post = {'evacuate': {'onSharedStorage': False}}
+        self.admin_api.post_server_action(servers[1]['id'], post)
+        evacuated_server = self._wait_for_state_change(servers[1], 'ACTIVE')
+
+        # Note(gibi): need to get the server again as the state of the instance
+        # goes to ACTIVE first then the host of the instance changes to the
+        # new host later
+        evacuated_server = self.admin_api.get_server(evacuated_server['id'])
+
+        host.start()
+
+        return [evacuated_server['OS-EXT-SRV-ATTR:host'],
+                servers[0]['OS-EXT-SRV-ATTR:host']]
+
+    def test_evacuate_with_soft_affinity(self):
+        evacuated_server, other_server = (
+            self._evacuate_with_soft_anti_affinity_policies(
+                self.soft_affinity))
+        self.assertNotEqual(evacuated_server, other_server)
+
+    def test_evacuate_with_soft_anti_affinity(self):
+        evacuated_server, other_server = (
+            self._evacuate_with_soft_anti_affinity_policies(
+                self.soft_anti_affinity))
+        self.assertEqual(evacuated_server, other_server)

nova/tests/unit/api/openstack/compute/test_versions.py

@@ -66,7 +66,7 @@ EXP_VERSIONS = {
     "v2.1": {
         "id": "v2.1",
         "status": "CURRENT",
-        "version": "2.14",
+        "version": "2.15",
         "min_version": "2.1",
         "updated": "2013-07-23T11:33:21Z",
         "links": [
@@ -128,7 +128,7 @@ class VersionsTestV20(test.NoDBTestCase):
             {
                 "id": "v2.1",
                 "status": "CURRENT",
-                "version": "2.14",
+                "version": "2.15",
                 "min_version": "2.1",
                 "updated": "2013-07-23T11:33:21Z",
                 "links": [