Expose a REST API for a specific list of RPs

Now that we merged the object method for getting the list of ResourceProviders
based on a specific amount request, we need to expose that method into a REST
API call so that the scheduler client could be calling it.

Co-Authored-By: Jay Pipes <jaypipes@gmail.com>

Change-Id: Ia8b534d20c064eb3a767f95ca22814925acfaa77
Implements: blueprint resource-providers-get-by-request

nova/api/openstack/placement/handlers/resource_provider.py

@@ -12,6 +12,7 @@
 """Placement API handlers for resource providers."""
 
 import copy
+import jsonschema
 
 from oslo_db import exception as db_exc
 from oslo_serialization import jsonutils
@@ -46,6 +47,102 @@ POST_RESOURCE_PROVIDER_SCHEMA = {
 PUT_RESOURCE_PROVIDER_SCHEMA = copy.deepcopy(POST_RESOURCE_PROVIDER_SCHEMA)
 PUT_RESOURCE_PROVIDER_SCHEMA['properties'].pop('uuid')
 
+# Represents the allowed query string parameters to the GET /resource_providers
+# API call
+GET_RPS_SCHEMA_1_0 = {
+    "type": "object",
+    "properties": {
+        "name": {
+            "type": "string"
+        },
+        "uuid": {
+            "type": "string",
+            "format": "uuid"
+        }
+    },
+    "additionalProperties": False,
+}
+
+# Placement API microversion 1.3 adds support for a member_of attribute
+GET_RPS_SCHEMA_1_3 = copy.deepcopy(GET_RPS_SCHEMA_1_0)
+GET_RPS_SCHEMA_1_3['properties']['member_of'] = {
+    # TODO(mriedem): At some point we need to do jsonschema and/or uuid
+    # validation of the value(s) here.
+    "type": "string"
+}
+
+# Placement API microversion 1.4 adds support for requesting resource providers
+# having some set of capacity for some resources. The query string is a
+# comma-delimited set of "$RESOURCE_CLASS_NAME:$AMOUNT" strings. The validation
+# of the string is left up to the helper code in the
+# _normalize_resources_qs_param() function below.
+GET_RPS_SCHEMA_1_4 = copy.deepcopy(GET_RPS_SCHEMA_1_3)
+GET_RPS_SCHEMA_1_4['properties']['resources'] = {
+    "type": "string"
+}
+
+
+def _normalize_resources_qs_param(qs):
+    """Given a query string parameter for resources, validate it meets the
+    expected format and return a dict of amounts, keyed by resource class name.
+
+    The expected format of the resources parameter looks like so:
+
+        $RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT
+
+    So, if the user was looking for resource providers that had room for an
+    instance that will consume 2 vCPUs, 1024 MB of RAM and 50GB of disk space,
+    they would use the following query string:
+
+        ?resources=VCPU:2,MEMORY_MB:1024:DISK_GB:50
+
+    The returned value would be:
+
+        {
+            "VCPU": 2,
+            "MEMORY_MB": 1024,
+            "DISK_GB": 50,
+        }
+
+    :param qs: The value of the 'resources' query string parameter
+    :raises `webob.exc.HTTPBadRequest` if the parameter's value isn't in the
+            expected format.
+    """
+    result = {}
+    resource_tuples = qs.split(',')
+    for rt in resource_tuples:
+        try:
+            rc_name, amount = rt.split(':')
+        except ValueError:
+            msg = _('Badly formed resources parameter. Expected resources '
+                    'query string parameter in form: '
+                    '?resources=VCPU:2,MEMORY_MB:1024. Got: %s.')
+            msg = msg % rt
+            raise webob.exc.HTTPBadRequest(msg,
+                    json_formatter=util.json_error_formatter)
+        try:
+            amount = int(amount)
+        except ValueError:
+            msg = _('Requested resource %(resource_name)s expected positive '
+                    'integer amount. Got: %(amount)s.')
+            msg = msg % {
+                'resource_name': rc_name,
+                'amount': amount,
+            }
+            raise webob.exc.HTTPBadRequest(msg,
+                    json_formatter=util.json_error_formatter)
+        if amount < 1:
+            msg = _('Requested resource %(resource_name)s requires '
+                    'amount >= 1. Got: %(amount)d.')
+            msg = msg % {
+                'resource_name': rc_name,
+                'amount': amount,
+            }
+            raise webob.exc.HTTPBadRequest(msg,
+                    json_formatter=util.json_error_formatter)
+        result[rc_name] = amount
+    return result
+
 
 def _serialize_links(environ, resource_provider):
     url = util.resource_provider_url(environ, resource_provider)
@@ -165,24 +262,22 @@ def list_resource_providers(req):
     context = req.environ['placement.context']
     want_version = req.environ[microversion.MICROVERSION_ENVIRON]
 
-    allowed_filters = set(objects.ResourceProviderList.allowed_filters)
+    schema = GET_RPS_SCHEMA_1_0
-    if not want_version.matches((1, 3)):
+    if want_version == (1, 3):
-        allowed_filters.remove('member_of')
+        schema = GET_RPS_SCHEMA_1_3
-    passed_filters = set(req.GET.keys())
+    if want_version >= (1, 4):
-    invalid_filters = passed_filters - allowed_filters
+        schema = GET_RPS_SCHEMA_1_4
-    if invalid_filters:
+    try:
+        jsonschema.validate(dict(req.GET), schema,
+                            format_checker=jsonschema.FormatChecker())
+    except jsonschema.ValidationError as exc:
         raise webob.exc.HTTPBadRequest(
-            _('Invalid filters: %(filters)s') %
+            _('Invalid query string parameters: %(exc)s') %
-            {'filters': ', '.join(invalid_filters)},
+            {'exc': exc},
-            json_formatter=util.json_error_formatter)
-
-    if 'uuid' in req.GET and not uuidutils.is_uuid_like(req.GET['uuid']):
-        raise webob.exc.HTTPBadRequest(
-            _('Invalid uuid value: %(uuid)s') % {'uuid': req.GET['uuid']},
             json_formatter=util.json_error_formatter)
 
     filters = {}
-    for attr in objects.ResourceProviderList.allowed_filters:
+    for attr in ['uuid', 'name', 'member_of']:
         if attr in req.GET:
             value = req.GET[attr]
             # special case member_of to always make its value a
@@ -196,8 +291,17 @@ def list_resource_providers(req):
                 else:
                     value = [value]
             filters[attr] = value
-    resource_providers = objects.ResourceProviderList.get_all_by_filters(
+    if 'resources' in req.GET:
-        context, filters)
+        resources = _normalize_resources_qs_param(req.GET['resources'])
+        filters['resources'] = resources
+    try:
+        resource_providers = objects.ResourceProviderList.get_all_by_filters(
+            context, filters)
+    except exception.ResourceClassNotFound as exc:
+        raise webob.exc.HTTPBadRequest(
+            _('Invalid resource class in resources parameter: %(error)s') %
+            {'error': exc},
+            json_formatter=util.json_error_formatter)
 
     response = req.response
     response.body = jsonutils.dumps(_serialize_providers(

nova/api/openstack/placement/microversion.py

@@ -39,6 +39,7 @@ VERSIONS = [
     '1.2',  # Adds /resource_classes resource endpoint
     '1.3',  # Adds 'member_of' query parameter to get resource providers
             # that are members of any of the listed aggregates
+    '1.4',  # Adds resources query string parameter in GET /resource_providers
 ]
 
 

nova/api/openstack/placement/rest_api_version_history.rst

@@ -52,3 +52,32 @@ Version 1.3 adds support for listing resource providers that are members of
 any of the list of aggregates provided using a ``member_of`` query parameter:
 
 * /resource_providers?member_of=in:{agg1_uuid},{agg2_uuid},{agg3_uuid}
+
+1.4 -- Filter resource providers having requested resource capacity
+-------------------------------------------------------------------
+
+The 1.4 version adds support for querying resource providers that have the
+ability to serve a requested set of resources. A new "resources" query string
+parameter is now accepted to the `GET /resource_providers` API call. This
+parameter indicates the requested amounts of various resources that a provider
+must have the capacity to serve. The "resources" query string parameter takes
+the form:
+
+``?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT``
+
+For instance, if the user wishes to see resource providers that can service a
+request for 2 vCPUs, 1024 MB of RAM and 50 GB of disk space, the user can issue
+a request to:
+
+`GET /resource_providers?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50`
+
+If the resource class does not exist, then it will return a HTTP 400.
+
+.. note:: The resources filtering is also based on the `min_unit`, `max_unit`
+    and `step_size` of the inventory record. For example, if the `max_unit` is
+    512 for the DISK_GB inventory for a particular resource provider and a
+    GET request is made for `DISK_GB:1024`, that resource provider will not be
+    returned. The `min_unit` is the minimum amount of resource that can be
+    requested for a given inventory and resource provider. The `step_size` is
+    the increment of resource that can be requested for a given resource on a
+    given provider.

nova/tests/functional/api/openstack/placement/fixtures.py

@@ -91,9 +91,12 @@ class AllocationFixture(APIFixture):
         rp = objects.ResourceProvider(
             self.context, name=rp_name, uuid=rp_uuid)
         rp.create()
+
+        # Create some DISK_GB inventory and allocations.
         inventory = objects.Inventory(
             self.context, resource_provider=rp,
-            resource_class='DISK_GB', total=2048)
+            resource_class='DISK_GB', total=2048,
+            step_size=10, min_unit=10, max_unit=600)
         inventory.obj_set_defaults()
         rp.add_inventory(inventory)
         allocation = objects.Allocation(
@@ -108,3 +111,28 @@ class AllocationFixture(APIFixture):
             consumer_id=uuidutils.generate_uuid(),
             used=512)
         allocation.create()
+
+        # Create some VCPU inventory and allocations.
+        inventory = objects.Inventory(
+            self.context, resource_provider=rp,
+            resource_class='VCPU', total=8,
+            max_unit=4)
+        inventory.obj_set_defaults()
+        rp.add_inventory(inventory)
+        allocation = objects.Allocation(
+            self.context, resource_provider=rp,
+            resource_class='VCPU',
+            consumer_id=uuidutils.generate_uuid(),
+            used=2)
+        allocation.create()
+        allocation = objects.Allocation(
+            self.context, resource_provider=rp,
+            resource_class='VCPU',
+            consumer_id=uuidutils.generate_uuid(),
+            used=4)
+        allocation.create()
+
+        # The ALT_RP_XXX variables are for a resource provider that has
+        # not been created in the Allocation fixture
+        os.environ['ALT_RP_UUID'] = uuidutils.generate_uuid()
+        os.environ['ALT_RP_NAME'] = uuidutils.generate_uuid()

nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml

@@ -37,13 +37,13 @@ tests:
   response_strings:
       - "Unacceptable version header: 0.5"
 
-- name: latest microversion is 1.3
+- name: latest microversion is 1.4
   GET: /
   request_headers:
       openstack-api-version: placement latest
   response_headers:
       vary: /OpenStack-API-Version/
-      openstack-api-version: placement 1.3
+      openstack-api-version: placement 1.4
 
 - name: other accept header bad version
   GET: /

nova/tests/functional/api/openstack/placement/gabbits/resource-provider-aggregates.yaml

@@ -90,11 +90,11 @@ tests:
   request_headers:
       openstack-api-version: placement 1.1
   status: 400
-  response_json_paths:
+  response_strings:
-      $.errors[0].detail: '/Invalid filters: member_of/'
+      - 'Invalid query string parameters'
 
 - name: error on bogus query parameter
   GET: '/resource_providers?assoc_with_aggregate=in:83a3d69d-8920-48e2-8914-cadfd8fa2f91,99652f11-9f77-46b9-80b7-4b1989be9f8c'
   status: 400
-  response_json_paths:
+  response_strings:
-      $.errors[0].detail: '/Invalid filters: assoc_with_aggregate/'
+      - 'Invalid query string parameters'

nova/tests/functional/api/openstack/placement/gabbits/resource-provider-resources-query.yaml

@@ -0,0 +1,136 @@
+
+fixtures:
+    - AllocationFixture
+
+defaults:
+    request_headers:
+        x-auth-token: admin
+        content-type: application/json
+        OpenStack-API-Version: placement latest
+
+tests:
+
+- name: what is at resource providers
+  GET: /resource_providers
+  response_json_paths:
+      $.resource_providers.`len`: 1
+      $.resource_providers[0].uuid: $ENVIRON['RP_UUID']
+      $.resource_providers[0].name: $ENVIRON['RP_NAME']
+      $.resource_providers[0].links[?rel = "self"].href: /resource_providers/$ENVIRON['RP_UUID']
+      $.resource_providers[0].links[?rel = "inventories"].href: /resource_providers/$ENVIRON['RP_UUID']/inventories
+      $.resource_providers[0].links[?rel = "aggregates"].href: /resource_providers/$ENVIRON['RP_UUID']/aggregates
+      $.resource_providers[0].links[?rel = "usages"].href: /resource_providers/$ENVIRON['RP_UUID']/usages
+
+- name: post new resource provider
+  POST: /resource_providers
+  data:
+      name: $ENVIRON['ALT_RP_NAME']
+      uuid: $ENVIRON['ALT_RP_UUID']
+  status: 201
+  response_headers:
+      location: //resource_providers/[a-f0-9-]+/
+  response_forbidden_headers:
+      - content-type
+
+- name: now 2 providers listed
+  GET: /resource_providers
+  response_json_paths:
+      $.resource_providers.`len`: 2
+
+- name: list resource providers providing resources filter before API 1.4
+  GET: /resource_providers?resources=VCPU:1
+  request_headers:
+      OpenStack-API-Version: placement 1.3
+  status: 400
+  response_strings:
+      - 'Invalid query string parameters'
+
+- name: list resource providers providing a badly-formatted resources filter
+  GET: /resource_providers?resources=VCPU
+  status: 400
+  response_strings:
+      - 'Badly formed resources parameter. Expected resources query string parameter in form:'
+      - 'Got: VCPU.'
+
+- name: list resource providers providing a resources filter with non-integer amount
+  GET: /resource_providers?resources=VCPU:fred
+  status: 400
+  response_strings:
+      - 'Requested resource VCPU expected positive integer amount.'
+      - 'Got: fred.'
+
+- name: list resource providers providing a resources filter with negative amount
+  GET: /resource_providers?resources=VCPU:-2
+  status: 400
+  response_strings:
+      - 'Requested resource VCPU requires amount >= 1.'
+      - 'Got: -2.'
+
+- name: list resource providers providing a resource class not existing
+  GET: /resource_providers?resources=MYMISSINGCLASS:1
+  status: 400
+  response_strings:
+      - 'Invalid resource class in resources parameter'
+
+- name: list resource providers providing a bad trailing comma
+  GET: /resource_providers?resources=DISK_GB:500,
+  status: 400
+  response_strings:
+      - 'Badly formed resources parameter. Expected resources query string parameter in form:'
+      # NOTE(mriedem): The value is empty because splitting on the trailing
+      # comma results in an empty string.
+      - 'Got: .'
+
+- name: list resource providers providing disk resources
+  GET: /resource_providers?resources=DISK_GB:500
+  response_json_paths:
+      $.resource_providers.`len`: 1
+      $.resource_providers[0].uuid: $ENVIRON['RP_UUID']
+
+- name: list resource providers providing disk and vcpu resources
+  GET: /resource_providers?resources=DISK_GB:500,VCPU:2
+  response_json_paths:
+      $.resource_providers.`len`: 1
+      $.resource_providers[0].uuid: $ENVIRON['RP_UUID']
+
+- name: list resource providers providing resources (no match - less than min_unit)
+  GET: /resource_providers?resources=DISK_GB:1
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: list resource providers providing resources (no match - more than max_unit)
+  GET: /resource_providers?resources=DISK_GB:610
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: list resource providers providing resources (no match - not enough inventory)
+  GET: /resource_providers?resources=DISK_GB:102400
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: list resource providers providing resources (no match - bad step size)
+  GET: /resource_providers?resources=DISK_GB:11
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: list resource providers providing resources (no match - no inventory of resource)
+  GET: /resource_providers?resources=MEMORY_MB:10240
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: list resource providers providing resources (no match - not enough VCPU)
+  GET: /resource_providers?resources=DISK_GB:500,VCPU:4
+  response_json_paths:
+      $.resource_providers.`len`: 0
+
+- name: associate an aggregate with rp1
+  PUT: /resource_providers/$ENVIRON['RP_UUID']/aggregates
+  data:
+      - 83a3d69d-8920-48e2-8914-cadfd8fa2f91
+  status: 200
+
+- name: get by aggregates with resources
+  GET: '/resource_providers?member_of=in:83a3d69d-8920-48e2-8914-cadfd8fa2f91&resources=VCPU:2'
+  response_json_paths:
+      $.resource_providers.`len`: 1
+      $.resource_providers[0].uuid: $ENVIRON['RP_UUID']

nova/tests/functional/api/openstack/placement/gabbits/resource-provider.yaml

@@ -129,13 +129,13 @@ tests:
   GET: /resource_providers?uuid=spameggs
   status: 400
   response_strings:
-      - 'Invalid uuid value: spameggs'
+      - 'Invalid query string parameters'
 
 - name: list resource providers providing an invalid filter
   GET: /resource_providers?spam=eggs
   status: 400
   response_strings:
-      - 'Invalid filters: spam'
+      - 'Invalid query string parameters'
 
 - name: list one resource provider filtering by uuid
   GET: /resource_providers?uuid=$ENVIRON['RP_UUID']

nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml

@@ -21,8 +21,9 @@ tests:
       # required but superfluous, is present
       content-type: /application/json/
   response_json_paths:
-      $.resource_provider_generation: 1
+      $.resource_provider_generation: 2
       $.usages.DISK_GB: 1024
+      $.usages.VCPU: 6
 
 - name: fail to delete resource provider
   DELETE: /resource_providers/$ENVIRON['RP_UUID']

releasenotes/notes/placement-rest-api-filter-providers-by-resources-0ab51c9766fe654f.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    A new Placement API microversion 1.4 is added. Users may now query the
+    Placement REST API for resource providers that have the ability to meet a
+    set of requested resource amounts. The `GET /resource_providers` API call
+    can have a "resources" query string parameter supplied that indicates the
+    requested amounts of various resources that a provider must have the
+    capacity to serve. The "resources" query string parameter takes the form:
+
+    ``?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT``
+
+    For instance, if the user wishes to see resource providers that can service
+    a request for 2 vCPUs, 1024 MB of RAM and 50 GB of disk space, the user can
+    issue a request of::
+
+    ``GET /resource_providers?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50``
+
+    The placement API is only available to admin users.