placement: support GET /allocation_candidates

A new 1.10 API microversion is added to return information that the
scheduler can use to select a particular set of resource providers to
claim resources for an instance.

The GET /allocation_candidates endpoint takes a "resources" querystring
parameter similar to the GET /resource_providers endpoint and returns a
dict with two top-level elements:

"allocation_requests" is a list of JSON objects that contain a
serialized HTTP body that the scheduler may subsequently use in a call
to PUT /allocations/{consumer_uuid} to claim resources against a
related set of resource providers.

"provider_summaries" is a JSON object, keyed by resource provider UUID,
of JSON objects of inventory/capacity information that the scheduler
can use to sort/weigh the results of the call when making its
destination host decisions.

Change-Id: I8dadb364746553d9495aa8bcffd0346ebc0b4baa
blueprint: placement-allocation-requests

nova/api/openstack/placement/handler.py

@@ -30,6 +30,7 @@ from oslo_log import log as logging
 
 from nova.api.openstack.placement.handlers import aggregate
 from nova.api.openstack.placement.handlers import allocation
+from nova.api.openstack.placement.handlers import allocation_candidate
 from nova.api.openstack.placement.handlers import inventory
 from nova.api.openstack.placement.handlers import resource_class
 from nova.api.openstack.placement.handlers import resource_provider
@@ -104,6 +105,9 @@ ROUTE_DECLARATIONS = {
         'PUT': allocation.set_allocations,
         'DELETE': allocation.delete_allocations,
     },
+    '/allocation_candidates': {
+        'GET': allocation_candidate.list_allocation_candidates,
+    },
     '/traits': {
         'GET': trait.list_traits,
     },

nova/api/openstack/placement/handlers/allocation_candidate.py

@@ -0,0 +1,183 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Placement API handlers for getting allocation candidates."""
+
+import collections
+
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
+import webob
+
+from nova.api.openstack.placement import microversion
+from nova.api.openstack.placement import util
+from nova.api.openstack.placement import wsgi_wrapper
+from nova import exception
+from nova.i18n import _
+from nova.objects import resource_provider as rp_obj
+
+
+LOG = logging.getLogger(__name__)
+
+# Represents the allowed query string parameters to the GET
+# /allocation_candidates API call
+_GET_SCHEMA_1_10 = {
+    "type": "object",
+    "properties": {
+        "resources": {
+            "type": "string"
+        },
+    },
+    "required": [
+        "resources",
+    ],
+    "additionalProperties": False,
+}
+
+
+def _transform_allocation_requests(alloc_reqs):
+    """Turn supplied list of AllocationRequest objects into a list of dicts of
+    resources involved in the allocation request. The returned results is
+    intended to be able to be used as the body of a PUT
+    /allocations/{consumer_uuid} HTTP request, so therefore we return a list of
+    JSON objects that looks like the following:
+
+    [
+        {
+            "allocations": [
+                {
+                    "resource_provider": {
+                        "uuid": $rp_uuid,
+                    }
+                    "resources": {
+                        $resource_class: $requested_amount, ...
+                    },
+                }, ...
+            ],
+        }, ...
+    ]
+    """
+    results = []
+    for ar in alloc_reqs:
+        provider_resources = collections.defaultdict(dict)
+        for rr in ar.resource_requests:
+            res_dict = provider_resources[rr.resource_provider.uuid]
+            res_dict[rr.resource_class] = rr.amount
+
+        allocs = [
+            {
+                "resource_provider": {
+                    "uuid": rp_uuid,
+                },
+                "resources": resources,
+            } for rp_uuid, resources in provider_resources.items()
+        ]
+        alloc = {
+            "allocations": allocs
+        }
+        results.append(alloc)
+    return results
+
+
+def _transform_provider_summaries(p_sums):
+    """Turn supplied list of ProviderSummary objects into a dict, keyed by
+    resource provider UUID, of dicts of provider and inventory information.
+
+    {
+       RP_UUID_1: {
+           'resources': {
+              'DISK_GB': {
+                'capacity': 100,
+                'used': 0,
+              },
+              'VCPU': {
+                'capacity': 4,
+                'used': 0,
+              }
+           }
+       },
+       RP_UUID_2: {
+           'resources': {
+              'DISK_GB': {
+                'capacity': 100,
+                'used': 0,
+              },
+              'VCPU': {
+                'capacity': 4,
+                'used': 0,
+              }
+           }
+       }
+    }
+    """
+    return {
+        ps.resource_provider.uuid: {
+            'resources': {
+                psr.resource_class: {
+                    'capacity': psr.capacity,
+                    'used': psr.used,
+                } for psr in ps.resources
+            }
+        } for ps in p_sums
+    }
+
+
+def _transform_allocation_candidates(alloc_cands):
+    """Turn supplied AllocationCandidates object into a dict containing
+    allocation requests and provider summaries.
+
+    {
+        'allocation_requests': <ALLOC_REQUESTS>,
+        'provider_summaries': <PROVIDER_SUMMARIES>,
+    }
+    """
+    a_reqs = _transform_allocation_requests(alloc_cands.allocation_requests)
+    p_sums = _transform_provider_summaries(alloc_cands.provider_summaries)
+    return {
+        'allocation_requests': a_reqs,
+        'provider_summaries': p_sums,
+    }
+
+
+@wsgi_wrapper.PlacementWsgify
+@microversion.version_handler('1.10')
+@util.check_accept('application/json')
+def list_allocation_candidates(req):
+    """GET a JSON object with a list of allocation requests and a JSON object
+    of provider summary objects
+
+    On success return a 200 and an application/json body representing
+    a collection of allocation requests and provider summaries
+    """
+    context = req.environ['placement.context']
+    schema = _GET_SCHEMA_1_10
+    util.validate_query_params(req, schema)
+
+    resources = util.normalize_resources_qs_param(req.GET['resources'])
+    filters = {
+        'resources': resources,
+    }
+
+    try:
+        cands = rp_obj.AllocationCandidates.get_by_filters(context, filters)
+    except exception.ResourceClassNotFound as exc:
+        raise webob.exc.HTTPBadRequest(
+            _('Invalid resource class in resources parameter: %(error)s') %
+            {'error': exc})
+
+    response = req.response
+    trx_cands = _transform_allocation_candidates(cands)
+    json_data = jsonutils.dumps(trx_cands)
+    response.body = encodeutils.to_utf8(json_data)
+    response.content_type = 'application/json'
+    return response

nova/api/openstack/placement/handlers/resource_provider.py

@@ -74,7 +74,7 @@ GET_RPS_SCHEMA_1_3['properties']['member_of'] = {
 # having some set of capacity for some resources. The query string is a
 # comma-delimited set of "$RESOURCE_CLASS_NAME:$AMOUNT" strings. The validation
 # of the string is left up to the helper code in the
-# _normalize_resources_qs_param() function below.
+# normalize_resources_qs_param() function.
 GET_RPS_SCHEMA_1_4 = copy.deepcopy(GET_RPS_SCHEMA_1_3)
 GET_RPS_SCHEMA_1_4['properties']['resources'] = {
     "type": "string"

nova/api/openstack/placement/microversion.py

@@ -47,6 +47,7 @@ VERSIONS = [
     '1.8',  # Adds 'project_id' and 'user_id' required request parameters to
             # PUT /allocations
     '1.9',  # Adds GET /usages
+    '1.10',  # Adds GET /allocation_candidates resource endpoint
 ]
 
 

nova/api/openstack/placement/rest_api_version_history.rst

@@ -144,3 +144,11 @@ The following new routes are added:
 ``GET /usages?project_id=<project_id>&user_id=<user_id>``
 
    Returns all usages for a given project and user.
+
+1.10 Allocation candidates
+-------------------------------------------
+
+The 1.10 version brings a new REST resource endpoint for getting a list of
+allocation candidates. Allocation candidates are collections of possible
+allocations against resource providers that can satisfy a particular request
+for resources.

nova/tests/functional/api/openstack/placement/fixtures.py

@@ -191,6 +191,104 @@ class AllocationFixture(APIFixture):
         os.environ['ALT_RP_NAME'] = uuidutils.generate_uuid()
 
 
+class SharedStorageFixture(APIFixture):
+    """An APIFixture that has some two compute nodes without local storage
+    associated by aggregate to a provider of shared storage.
+    """
+
+    def start_fixture(self):
+        super(SharedStorageFixture, self).start_fixture()
+        self.context = context.get_admin_context()
+
+        # These UUIDs are staticly defined here because the JSONPath querying
+        # needed in the allocation-candidates.yaml gabbits cannot refer to an
+        # ENVIRON variable because the $ sign is a token in the JSONPath
+        # parser.
+        os.environ['CN1_UUID'] = 'c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'
+        os.environ['CN2_UUID'] = 'c2c2c2c2-beef-49a0-98a0-b998b88debfd'
+        os.environ['SS_UUID'] = 'dddddddd-61a6-472e-b8c1-74796e803066'
+        os.environ['AGG_UUID'] = 'aaaaaaaa-04b3-458c-9a9f-361aad56f41c'
+
+        cn1_uuid = os.environ['CN1_UUID']
+        cn2_uuid = os.environ['CN2_UUID']
+        ss_uuid = os.environ['SS_UUID']
+        agg_uuid = os.environ['AGG_UUID']
+
+        cn1 = objects.ResourceProvider(
+            self.context,
+            name='cn1',
+            uuid=cn1_uuid)
+        cn1.create()
+
+        cn2 = objects.ResourceProvider(
+            self.context,
+            name='cn2',
+            uuid=cn2_uuid)
+        cn2.create()
+
+        ss = objects.ResourceProvider(
+            self.context,
+            name='ss',
+            uuid=ss_uuid)
+        ss.create()
+
+        # Populate compute node inventory for VCPU and RAM
+        for cn in (cn1, cn2):
+            vcpu_inv = objects.Inventory(
+                self.context,
+                resource_provider=cn,
+                resource_class='VCPU',
+                total=24,
+                reserved=0,
+                max_unit=24,
+                min_unit=1,
+                step_size=1,
+                allocation_ratio=16.0)
+            vcpu_inv.obj_set_defaults()
+            ram_inv = objects.Inventory(
+                self.context,
+                resource_provider=cn,
+                resource_class='MEMORY_MB',
+                total=128 * 1024,
+                reserved=0,
+                max_unit=128 * 1024,
+                min_unit=256,
+                step_size=256,
+                allocation_ratio=1.5)
+            ram_inv.obj_set_defaults()
+            inv_list = objects.InventoryList(objects=[vcpu_inv, ram_inv])
+            cn.set_inventory(inv_list)
+
+        # Populate shared storage provider with DISK_GB inventory
+        disk_inv = objects.Inventory(
+            self.context,
+            resource_provider=ss,
+            resource_class='DISK_GB',
+            total=2000,
+            reserved=100,
+            max_unit=2000,
+            min_unit=10,
+            step_size=10,
+            allocation_ratio=1.0)
+        disk_inv.obj_set_defaults()
+        inv_list = objects.InventoryList(objects=[disk_inv])
+        ss.set_inventory(inv_list)
+
+        # Mark the shared storage pool as having inventory shared among any
+        # provider associated via aggregate
+        t = objects.Trait.get_by_name(
+            self.context,
+            "MISC_SHARES_VIA_AGGREGATE",
+        )
+        ss.set_traits(objects.TraitList(objects=[t]))
+
+        # Now associate the shared storage pool and both compute nodes with the
+        # same aggregate
+        cn1.set_aggregates([agg_uuid])
+        cn2.set_aggregates([agg_uuid])
+        ss.set_aggregates([agg_uuid])
+
+
 class CORSFixture(APIFixture):
     """An APIFixture that turns on CORS."""
 

nova/tests/functional/api/openstack/placement/gabbits/allocation-candidates.yaml

@@ -0,0 +1,72 @@
+# Tests of allocation candidates API
+
+fixtures:
+    - SharedStorageFixture
+
+defaults:
+    request_headers:
+        x-auth-token: admin
+        accept: application/json
+        openstack-api-version: placement 1.10
+
+tests:
+
+# NOTE(jaypipes): The following static UUIDs are used in this file. We use
+# static UUIDs because JSONPath's parser cannot understand $ subtitution if we
+# refer to them with $ENVIRON[]
+#
+#  os.environ['CN1_UUID'] = 'c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'
+#  os.environ['CN2_UUID'] = 'c2c2c2c2-beef-49a0-98a0-b998b88debfd'
+#  os.environ['SS_UUID'] = 'dddddddd-61a6-472e-b8c1-74796e803066'
+#  os.environ['AGG_UUID'] = 'aaaaaaaa-04b3-458c-9e9f-361aad56f41c'
+
+- name: get allocation candidates before microversion
+  GET: /allocation_candidates?resources=VCPU:1
+  request_headers:
+      openstack-api-version: placement 1.8
+  status: 404
+
+- name: get allocation candidates no resources
+  GET: /allocation_candidates
+  status: 400
+  response_strings:
+      - "'resources' is a required property"
+
+- name: get allocation candidates no allocations yet
+  GET: /allocation_candidates?resources=VCPU:1,MEMORY_MB:1024,DISK_GB:100
+  status: 200
+  response_json_paths:
+      # There are 3 providers involved. 2 compute nodes, 1 shared storage
+      # provider
+      $.provider_summaries.`len`: 3
+      # However, there are only 2 allocation requests, one for each compute
+      # node that provides the VCPU/MEMORY_MB and DISK_GB provided by the
+      # shared storage provider
+      $.allocation_requests.`len`: 2
+      # Verify that compute node #1 only has VCPU and MEMORY_MB listed in the
+      # resource requests
+      $.allocation_requests..allocations[?(@.resource_provider.uuid='c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d')].resources[VCPU]: 1
+      $.allocation_requests..allocations[?(@.resource_provider.uuid='c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d')].resources[MEMORY_MB]: 1024
+      # Verify that compute node #2 only has VCPU and MEMORY_MB listed in the
+      # resource requests
+      $.allocation_requests..allocations[?(@.resource_provider.uuid='c2c2c2c2-beef-49a0-98a0-b998b88debfd')].resources[VCPU]: 1
+      $.allocation_requests..allocations[?(@.resource_provider.uuid='c2c2c2c2-beef-49a0-98a0-b998b88debfd')].resources[MEMORY_MB]: 1024
+      # Verify that shared storage provider only has DISK_GB listed in the
+      # resource requests, but is listed twice
+      $.allocation_requests..allocations[?(@.resource_provider.uuid='dddddddd-61a6-472e-b8c1-74796e803066')].resources[DISK_GB]: [100, 100]
+      # Verify that the resources listed in the provider summary for compute
+      # node #1 show correct capacity and usage
+      $.provider_summaries['c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'].resources[VCPU].capacity: 384 # 16.0 * 24
+      $.provider_summaries['c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'].resources[VCPU].used: 0
+      $.provider_summaries['c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'].resources[MEMORY_MB].capacity: 196608 # 1.5 * 128G
+      $.provider_summaries['c1c1c1c1-2894-4df1-aa6b-c61fa72ed22d'].resources[MEMORY_MB].used: 0
+      # Verify that the resources listed in the provider summary for compute
+      # node #2 show correct capacity and usage
+      $.provider_summaries['c2c2c2c2-beef-49a0-98a0-b998b88debfd'].resources[VCPU].capacity: 384 # 16.0 * 24
+      $.provider_summaries['c2c2c2c2-beef-49a0-98a0-b998b88debfd'].resources[VCPU].used: 0
+      $.provider_summaries['c2c2c2c2-beef-49a0-98a0-b998b88debfd'].resources[MEMORY_MB].capacity: 196608 # 1.5 * 128G
+      $.provider_summaries['c2c2c2c2-beef-49a0-98a0-b998b88debfd'].resources[MEMORY_MB].used: 0
+      # Verify that the resources listed in the provider summary for shared
+      # storage show correct capacity and usage
+      $.provider_summaries['dddddddd-61a6-472e-b8c1-74796e803066'].resources[DISK_GB].capacity: 1900 # 1.0 * 2000 - 100G
+      $.provider_summaries['dddddddd-61a6-472e-b8c1-74796e803066'].resources[DISK_GB].used: 0

nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml

@@ -39,13 +39,13 @@ tests:
   response_json_paths:
       $.errors[0].title: Not Acceptable
 
-- name: latest microversion is 1.9
+- name: latest microversion is 1.10
   GET: /
   request_headers:
       openstack-api-version: placement latest
   response_headers:
       vary: /OpenStack-API-Version/
-      openstack-api-version: placement 1.9
+      openstack-api-version: placement 1.10
 
 - name: other accept header bad version
   GET: /

nova/tests/unit/api/openstack/placement/test_microversion.py

@@ -74,7 +74,7 @@ class TestMicroversionIntersection(test.NoDBTestCase):
     # if you add two different versions of method 'foobar' the
     # number only goes up by one if no other version foobar yet
     # exists. This operates as a simple sanity check.
-    TOTAL_VERSIONED_METHODS = 14
+    TOTAL_VERSIONED_METHODS = 15
 
     def test_methods_versioned(self):
         methods_data = microversion.VERSIONED_METHODS

releasenotes/notes/placement-allocation-candidates-1114a843755b93c4.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    A new 1.10 API microversion is added to the Placement REST API. This
+    microversion adds support for the GET /allocation_candidates resource
+    endpoint. This endpoint returns information about possible allocation
+    requests that callers can make which meet a set of resource constraints
+    supplied as query string parameters. Also returned is some inventory and
+    capacity information for the resource providers involved in the allocation
+    candidates.