Merge "Add support for inventories to placement API"

nova/api/openstack/placement/handler.py

@@ -26,6 +26,7 @@ method.
 import routes
 import webob
 
+from nova.api.openstack.placement.handlers import inventory
 from nova.api.openstack.placement.handlers import resource_provider
 from nova.api.openstack.placement.handlers import root
 from nova.api.openstack.placement import util
@@ -50,6 +51,16 @@ ROUTE_DECLARATIONS = {
         'DELETE': resource_provider.delete_resource_provider,
         'PUT': resource_provider.update_resource_provider
     },
+    '/resource_providers/{uuid}/inventories': {
+        'GET': inventory.get_inventories,
+        'POST': inventory.create_inventory,
+        'PUT': inventory.set_inventories
+    },
+    '/resource_providers/{uuid}/inventories/{resource_class}': {
+        'GET': inventory.get_inventory,
+        'PUT': inventory.update_inventory,
+        'DELETE': inventory.delete_inventory
+    },
 }
 
 

nova/api/openstack/placement/handlers/inventory.py

@@ -0,0 +1,408 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Inventory handlers for Placement API."""
+
+import copy
+
+import jsonschema
+from oslo_db import exception as db_exc
+from oslo_serialization import jsonutils
+import webob
+
+from nova.api.openstack.placement import util
+from nova import exception
+from nova import objects
+
+
+BASE_INVENTORY_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "resource_provider_generation": {
+            "type": "integer"
+        },
+        "total": {
+            "type": "integer"
+        },
+        "reserved": {
+            "type": "integer"
+        },
+        "min_unit": {
+            "type": "integer"
+        },
+        "max_unit": {
+            "type": "integer"
+        },
+        "step_size": {
+            "type": "integer"
+        },
+        "allocation_ratio": {
+            "type": "number"
+        },
+    },
+    "required": [
+        "total",
+        "resource_provider_generation"
+    ],
+    "additionalProperties": False
+}
+POST_INVENTORY_SCHEMA = copy.deepcopy(BASE_INVENTORY_SCHEMA)
+POST_INVENTORY_SCHEMA['properties']['resource_class'] = {
+    "type": "string",
+    "pattern": "^[A-Z0-9_]+$"
+}
+POST_INVENTORY_SCHEMA['required'].append('resource_class')
+POST_INVENTORY_SCHEMA['required'].remove('resource_provider_generation')
+PUT_INVENTORY_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "resource_provider_generation": {
+            "type": "integer"
+        },
+        "inventories": {
+            "type": "array",
+            "items": POST_INVENTORY_SCHEMA
+        }
+    },
+    "required": [
+        "resource_provider_generation",
+        "inventories"
+    ],
+    "additionalProperties": False
+}
+
+# NOTE(cdent): We keep our own representation of inventory defaults
+# and output fields, separate from the versioned object to avoid
+# inadvertent API changes when the object defaults are changed.
+OUTPUT_INVENTORY_FIELDS = [
+    'total',
+    'reserved',
+    'min_unit',
+    'max_unit',
+    'step_size',
+    'allocation_ratio',
+]
+INVENTORY_DEFAULTS = {
+    'reserved': 0,
+    'min_unit': 0,
+    'max_unit': 0,
+    'step_size': 1,
+    'allocation_ratio': 1.0
+}
+
+
+def _extract_json(body, schema):
+    """Extract and validate data from JSON body."""
+    try:
+        data = jsonutils.loads(body)
+    except ValueError as exc:
+        raise webob.exc.HTTPBadRequest(
+            'Malformed JSON: %s' % exc,
+            json_formatter=util.json_error_formatter)
+    try:
+        jsonschema.validate(data, schema)
+    except jsonschema.ValidationError as exc:
+        raise webob.exc.HTTPBadRequest(
+            'JSON does not validate: %s' % exc,
+            json_formatter=util.json_error_formatter)
+    return data
+
+
+def _extract_inventory(body, schema):
+    """Extract and validate inventory from JSON body."""
+    data = _extract_json(body, schema)
+
+    inventory_data = copy.copy(INVENTORY_DEFAULTS)
+    inventory_data.update(data)
+
+    return inventory_data
+
+
+def _extract_inventories(body, schema):
+    """Extract and validate multiple inventories from JSON body."""
+    data = _extract_json(body, schema)
+
+    inventories = []
+    for raw_inventory in data['inventories']:
+        inventory_data = copy.copy(INVENTORY_DEFAULTS)
+        inventory_data.update(raw_inventory)
+        inventories.append(inventory_data)
+
+    data['inventories'] = inventories
+    return data
+
+
+def _make_inventory_object(resource_provider, **data):
+    """Single place to catch malformed Inventories."""
+    # TODO(cdent): Some of the validation checks that are done here
+    # could be done via JSONschema (using, for example, "minimum":
+    # 0) for non-negative integers. It's not clear if that is
+    # duplication or decoupling so leaving it as this for now.
+    try:
+        inventory = objects.Inventory(
+            resource_provider=resource_provider, **data)
+    except (ValueError, TypeError) as exc:
+        raise webob.exc.HTTPBadRequest(
+            'Bad inventory %s for resource provider %s: %s'
+            % (data['resource_class'], resource_provider.uuid, exc),
+            json_formatter=util.json_error_formatter)
+    return inventory
+
+
+def _send_inventories(response, resource_provider, inventories):
+    """Send a JSON representation of a list of inventories."""
+    response.status = 200
+    response.body = jsonutils.dumps(_serialize_inventories(
+        resource_provider.generation, inventories))
+    response.content_type = 'application/json'
+    return response
+
+
+def _send_inventory(response, resource_provider, inventory, status=200):
+    """Send a JSON representation of one single inventory."""
+    response.status = status
+    response.body = jsonutils.dumps(_serialize_inventory(
+        resource_provider.generation, inventory))
+    response.content_type = 'application/json'
+    return response
+
+
+def _serialize_inventory(generation, inventory):
+    """Turn a single inventory into a dictionary."""
+    data = {
+        field: getattr(inventory, field)
+        for field in OUTPUT_INVENTORY_FIELDS
+    }
+    data['resource_provider_generation'] = generation
+    return data
+
+
+def _serialize_inventories(generation, inventories):
+    """Turn a list of inventories in a dict by resource class."""
+    inventories_by_class = {inventory.resource_class: inventory
+                            for inventory in inventories}
+    inventories_dict = {}
+    for resource_class, inventory in inventories_by_class.items():
+        inventories_dict[resource_class] = _serialize_inventory(
+            generation, inventory)
+    return {'inventories': inventories_dict}
+
+
+@webob.dec.wsgify
+@util.require_content('application/json')
+def create_inventory(req):
+    """POST to create one inventory.
+
+    On success return a 201 response, a location header pointing
+    to the newly created inventory and an application/json representation
+    of the inventory.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+    data = _extract_inventory(req.body, POST_INVENTORY_SCHEMA)
+
+    inventory = _make_inventory_object(resource_provider, **data)
+
+    try:
+        resource_provider.add_inventory(inventory)
+    except (exception.ConcurrentUpdateDetected,
+            db_exc.DBDuplicateEntry) as exc:
+        raise webob.exc.HTTPConflict(
+            'Update conflict: %s' % exc,
+            json_formatter=util.json_error_formatter)
+    except exception.InvalidInventoryCapacity as exc:
+        raise webob.exc.HTTPBadRequest(
+            'Unable to create inventory for resource provider %s: %s'
+            % (resource_provider.uuid, exc),
+            json_formatter=util.json_error_formatter)
+
+    response = req.response
+    response.location = util.inventory_url(
+        req.environ, resource_provider, data['resource_class'])
+    return _send_inventory(response, resource_provider, inventory,
+                           status=201)
+
+
+@webob.dec.wsgify
+def delete_inventory(req):
+    """DELETE to destroy a single inventory.
+
+    If the inventory is in use or resource provider generation is out
+    of sync return a 409.
+
+    On success return a 204 and an empty body.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_class = util.wsgi_path_item(req.environ, 'resource_class')
+
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+    try:
+        resource_provider.delete_inventory(resource_class)
+    except (exception.ConcurrentUpdateDetected,
+            exception.InventoryInUse) as exc:
+        raise webob.exc.HTTPConflict(
+            'Unable to delete inventory of class %s: %s' % (
+                resource_class, exc),
+            json_formatter=util.json_error_formatter)
+
+    response = req.response
+    response.status = 204
+    response.content_type = None
+    return response
+
+
+@webob.dec.wsgify
+@util.check_accept('application/json')
+def get_inventories(req):
+    """GET a list of inventories.
+
+    On success return a 200 with an application/json body representing
+    a collection of inventories.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+    inventories = objects.InventoryList.get_all_by_resource_provider_uuid(
+        context, resource_provider.uuid)
+
+    return _send_inventories(req.response, resource_provider, inventories)
+
+
+@webob.dec.wsgify
+@util.check_accept('application/json')
+def get_inventory(req):
+    """GET one inventory.
+
+    On success return a 200 an application/json body representing one
+    inventory.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_class = util.wsgi_path_item(req.environ, 'resource_class')
+
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+    inventory = objects.InventoryList.get_all_by_resource_provider_uuid(
+        context, resource_provider.uuid).find(resource_class)
+
+    if not inventory:
+        raise webob.exc.HTTPNotFound(
+            'No inventory of class %s for %s'
+            % (resource_class, resource_provider.uuid),
+            json_formatter=util.json_error_formatter)
+
+    return _send_inventory(req.response, resource_provider, inventory)
+
+
+@webob.dec.wsgify
+@util.require_content('application/json')
+def set_inventories(req):
+    """PUT to set all inventory for a resource provider.
+
+    Create, update and delete inventory as required to reset all
+    the inventory.
+
+    If the resource generation is out of sync, return a 409.
+    If an inventory to be deleted is in use, return a 409.
+    If an inventory to be updated would set capacity to exceed existing
+    use, return a 409.
+    If any inventory to be created or updated has settings which are
+    invalid (for example reserved exceeds capacity), return a 400.
+
+    On success return a 200 with an application/json body representing
+    the inventories.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+
+    data = _extract_inventories(req.body, PUT_INVENTORY_SCHEMA)
+    if data['resource_provider_generation'] != resource_provider.generation:
+        raise webob.exc.HTTPConflict(
+            'resource provider generation conflict',
+            json_formatter=util.json_error_formatter)
+
+    inv_list = []
+    for inventory_data in data['inventories']:
+        inventory = _make_inventory_object(
+            resource_provider, **inventory_data)
+        inv_list.append(inventory)
+    inventories = objects.InventoryList(objects=inv_list)
+
+    try:
+        resource_provider.set_inventory(inventories)
+    except (exception.ConcurrentUpdateDetected,
+            exception.InventoryInUse,
+            exception.InvalidInventoryNewCapacityExceeded,
+            db_exc.DBDuplicateEntry) as exc:
+        raise webob.exc.HTTPConflict(
+            'update conflict: %s' % exc,
+            json_formatter=util.json_error_formatter)
+    except exception.InvalidInventoryCapacity as exc:
+        raise webob.exc.HTTPBadRequest(
+            'Unable to update inventory for resource provider %s: %s'
+            % (resource_provider.uuid, exc),
+            json_formatter=util.json_error_formatter)
+
+    return _send_inventories(req.response, resource_provider, inventories)
+
+
+@webob.dec.wsgify
+@util.require_content('application/json')
+def update_inventory(req):
+    """PUT to update one inventory.
+
+    If the resource generation is out of sync, return a 409.
+    If the inventory would set capacity to exceed existing use, return
+    a 409.
+    If the inventory has settings which are invalid (for example
+    reserved exceeds capacity), return a 400.
+
+    On success return a 200 with an application/json body representing
+    the inventory.
+    """
+    context = req.environ['placement.context']
+    uuid = util.wsgi_path_item(req.environ, 'uuid')
+    resource_class = util.wsgi_path_item(req.environ, 'resource_class')
+
+    resource_provider = objects.ResourceProvider.get_by_uuid(
+        context, uuid)
+
+    data = _extract_inventory(req.body, BASE_INVENTORY_SCHEMA)
+    if data['resource_provider_generation'] != resource_provider.generation:
+        raise webob.exc.HTTPConflict(
+            'resource provider generation conflict',
+            json_formatter=util.json_error_formatter)
+
+    data['resource_class'] = resource_class
+    inventory = _make_inventory_object(resource_provider, **data)
+
+    try:
+        resource_provider.update_inventory(inventory)
+    except (exception.ConcurrentUpdateDetected,
+            exception.InvalidInventoryNewCapacityExceeded,
+            db_exc.DBDuplicateEntry) as exc:
+        raise webob.exc.HTTPConflict(
+            'update conflict: %s' % exc,
+            json_formatter=util.json_error_formatter)
+    except exception.InvalidInventoryCapacity as exc:
+        raise webob.exc.HTTPBadRequest(
+            'Unable to update inventory for resource provider %s: %s'
+            % (resource_provider.uuid, exc),
+            json_formatter=util.json_error_formatter)
+
+    return _send_inventory(req.response, resource_provider, inventory)

nova/api/openstack/placement/util.py

@@ -58,6 +58,13 @@ def check_accept(*types):
     return decorator
 
 
+def inventory_url(environ, resource_provider, resource_class=None):
+    url = '%s/inventories' % resource_provider_url(environ, resource_provider)
+    if resource_class:
+        url = '%s/%s' % (url, resource_class)
+    return url
+
+
 def json_error_formatter(body, status, title, environ):
     """A json_formatter for webob exceptions.
 

nova/tests/functional/api/openstack/placement/gabbits/inventory.yaml

@@ -0,0 +1,306 @@
+fixtures:
+    - APIFixture
+
+defaults:
+    request_headers:
+        x-auth-token: admin
+
+tests:
+- name: inventories for missing provider
+  GET: /resource_providers/7260669a-e3d4-4867-aaa7-683e2ab6958c/inventories
+  status: 404
+
+- name: post new resource provider
+  POST: /resource_providers
+  request_headers:
+    content-type: application/json
+  data:
+      name: $ENVIRON['RP_NAME']
+      uuid: $ENVIRON['RP_UUID']
+  status: 201
+  response_headers:
+      location: //resource_providers/[a-f0-9-]+/
+
+- name: get empty inventories
+  GET: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  response_json_paths:
+      $.inventories: {}
+
+- name: post an conflicting capacity inventory
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: DISK_GB
+      total: 256
+      reserved: 512
+  status: 400
+  response_strings:
+      - Unable to create inventory for resource provider
+
+- name: post an inventory
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: DISK_GB
+      total: 2048
+      reserved: 512
+      min_unit: 10
+      max_unit: 1024
+      step_size: 10
+      allocation_ratio: 1.0
+  status: 201
+  response_headers:
+      location: $SCHEME://$NETLOC/resource_providers/$ENVIRON['RP_UUID']/inventories/DISK_GB
+  response_json_paths:
+      $.total: 2048
+      $.reserved: 512
+
+- name: get that inventory
+  GET: $LOCATION
+  status: 200
+  response_json_paths:
+      $.resource_provider_generation: 1
+      $.total: 2048
+      $.reserved: 512
+      $.min_unit: 10
+      $.max_unit: 1024
+      $.step_size: 10
+      $.allocation_ratio: 1.0
+
+- name: modify the inventory
+  PUT: $LAST_URL
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 1
+      total: 2048
+      reserved: 1024
+      min_unit: 10
+      max_unit: 1024
+      step_size: 10
+      allocation_ratio: 1.0
+  status: 200
+  response_headers:
+      content-type: /application/json/
+  response_json_paths:
+      $.reserved: 1024
+
+- name: confirm inventory change
+  GET: $LAST_URL
+  response_json_paths:
+      $.resource_provider_generation: 2
+      $.total: 2048
+      $.reserved: 1024
+
+- name: modify inventory invalid generation
+  PUT: $LAST_URL
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 5
+      total: 2048
+  status: 409
+  response_strings:
+      - resource provider generation conflict
+
+- name: modify inventory invalid data
+  desc: This should 400 because reserved is greater than total
+  PUT: $LAST_URL
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 2
+      total: 2048
+      reserved: 4096
+      min_unit: 10
+      max_unit: 1024
+      step_size: 10
+      allocation_ratio: 1.0
+  status: 400
+  response_strings:
+      - Unable to update inventory for resource provider $ENVIRON['RP_UUID']
+
+- name: put inventory bad form
+  desc: This should 400 because reserved is greater than total
+  PUT: $LAST_URL
+  request_headers:
+      content-type: application/json
+  data:
+      house: red
+      car: blue
+  status: 400
+  response_strings:
+      - JSON does not validate
+
+- name: post inventory malformed json
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data: '{"foo": }'
+  status: 400
+  response_strings:
+      - Malformed JSON
+
+- name: post inventory bad syntax schema
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: bad_class
+      total: 2048
+  status: 400
+
+- name: post inventory bad resource class
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: NO_CLASS_14
+      total: 2048
+  status: 400
+
+- name: post inventory duplicated resource class
+  desc: DISK_GB was already created above
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: DISK_GB
+      total: 2048
+  status: 409
+  response_strings:
+      - Update conflict
+
+- name: get list of inventories
+  GET: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  response_json_paths:
+      $.inventories.DISK_GB.total: 2048
+      $.inventories.DISK_GB.reserved: 1024
+
+- name: delete the inventory
+  DELETE: /resource_providers/$ENVIRON['RP_UUID']/inventories/DISK_GB
+  status: 204
+
+- name: get now empty inventories
+  GET: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  response_json_paths:
+      $.inventories: {}
+
+- name: post new disk inventory
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: DISK_GB
+      total: 1024
+  status: 201
+
+- name: post new ipv4 address inventory
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: IPV4_ADDRESS
+      total: 255
+      reserved: 2
+  status: 201
+
+- name: list both those inventories
+  GET: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  response_json_paths:
+      $.inventories.DISK_GB.total: 1024
+      $.inventories.IPV4_ADDRESS.total: 255
+
+- name: post ipv4 address inventory again
+  POST: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_class: IPV4_ADDRESS
+      total: 255
+      reserved: 2
+  status: 409
+
+- name: delete inventory
+  DELETE: /resource_providers/$ENVIRON['RP_UUID']/inventories/IPV4_ADDRESS
+  status: 204
+  response_forbidden_headers:
+      - content-type
+
+- name: delete inventory again
+  DELETE: /resource_providers/$ENVIRON['RP_UUID']/inventories/IPV4_ADDRESS
+  status: 404
+
+- name: get missing inventory class
+  GET: /resource_providers/$ENVIRON['RP_UUID']/inventories/IPV4_ADDRESS
+  status: 404
+
+- name: create another resource provider
+  POST: /resource_providers
+  request_headers:
+    content-type: application/json
+  data:
+      name: disk-network
+  status: 201
+
+- name: put all inventory
+  PUT: $LOCATION/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 0
+      inventories:
+        - resource_class: IPV4_ADDRESS
+          total: 253
+        - resource_class: DISK_GB
+          total: 1024
+  status: 200
+  response_json_paths:
+      $.inventories.IPV4_ADDRESS.total: 253
+      $.inventories.IPV4_ADDRESS.reserved: 0
+      $.inventories.DISK_GB.total: 1024
+      $.inventories.DISK_GB.allocation_ratio: 1.0
+
+- name: check both inventory classes
+  GET: $LAST_URL
+  response_json_paths:
+      $.inventories.DISK_GB.total: 1024
+      $.inventories.IPV4_ADDRESS.total: 253
+
+- name: check one inventory class
+  GET: $LAST_URL/DISK_GB
+  response_json_paths:
+      $.total: 1024
+
+- name: put all inventory bad generation
+  PUT: /resource_providers/$ENVIRON['RP_UUID']/inventories
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 99
+      inventories:
+        - resource_class: IPV4_ADDRESS
+          total: 253
+  status: 409
+  response_strings:
+      - resource provider generation conflict
+
+# NOTE(cdent): The generation is 6 now, based on the activity at
+# the start of this file.
+- name: put all inventory bad capacity
+  PUT: $LAST_URL
+  request_headers:
+      content-type: application/json
+  data:
+      resource_provider_generation: 6
+      inventories:
+        - resource_class: IPV4_ADDRESS
+          total: 253
+          reserved: 512
+  status: 400
+  response_strings:
+      - Unable to update inventory

nova/tests/unit/api/openstack/placement/test_util.py

@@ -203,3 +203,18 @@ class TestPlacementURLs(test.NoDBTestCase):
                         % uuidsentinel.rp_uuid)
         self.assertEqual(expected_url, util.resource_provider_url(
             environ, self.resource_provider))
+
+    def test_inventories_url(self):
+        environ = {}
+        expected_url = ('/resource_providers/%s/inventories'
+                        % uuidsentinel.rp_uuid)
+        self.assertEqual(expected_url, util.inventory_url(
+            environ, self.resource_provider))
+
+    def test_inventory_url(self):
+        resource_class = 'DISK_GB'
+        environ = {}
+        expected_url = ('/resource_providers/%s/inventories/%s'
+                        % (uuidsentinel.rp_uuid, resource_class))
+        self.assertEqual(expected_url, util.inventory_url(
+            environ, self.resource_provider, resource_class))