From 37721325798dda64d96d174f9e316620639fc9c9 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 30 Aug 2019 13:00:57 +0100 Subject: [PATCH] Microversion 1.38: API support for consumer types Update allocations, reshaper and usage APIs to accept and present consumer_type in microversion 1.38. ensure_consumer in placement/handlers/util.py is updated to be consumer type aware. allocation, usage and reshaper schema and handlers are updated gabbits/consumer-types-1.38.yaml adds tests across the various URIs A TODO is left in placement/handlers/allocation.py where the database is being accessed in a way that is not ideal. This will be cleared up in a followup patch (to add use of an AttributeCache). Co-Authored-By: Surya Seetharaman Co-Authored-By: melanie witt Story: 2005473 Task: 36421 Change-Id: I24c2315093e07dbf25c4fb53152e6a4de7477a51 --- api-ref/source/allocations.inc | 27 +- api-ref/source/parameters.yaml | 38 +++ api-ref/source/reshaper.inc | 1 + .../allocations/get-allocations-1.38.json | 22 ++ .../manage-allocations-request-1.38.json | 41 +++ .../update-allocations-request-1.38.json | 20 ++ .../samples/reshaper/post-reshaper-1.38.json | 71 +++++ .../samples/usages/get-usages-1.38.json | 22 ++ api-ref/source/usages.inc | 24 +- placement/handlers/allocation.py | 59 +++- placement/handlers/reshaper.py | 4 +- placement/handlers/usage.py | 35 ++- placement/handlers/util.py | 135 ++++++--- placement/microversion.py | 10 + placement/objects/usage.py | 35 ++- placement/rest_api_version_history.rst | 22 +- placement/schemas/allocation.py | 15 + placement/schemas/common.py | 2 + placement/schemas/reshaper.py | 5 + placement/schemas/usage.py | 17 ++ .../gabbits/allocations-legacy-rbac.yaml | 20 +- .../gabbits/allocations-policy.yaml | 4 +- .../gabbits/allocations-secure-rbac.yaml | 20 +- .../gabbits/consumer-types-1.38.yaml | 265 ++++++++++++++++++ .../functional/gabbits/microversion.yaml | 4 +- placement/tests/unit/handlers/test_util.py | 85 +++++- .../notes/consumer_type-857b812aef10381e.yaml | 18 ++ 27 files changed, 929 insertions(+), 92 deletions(-) create mode 100644 api-ref/source/samples/allocations/get-allocations-1.38.json create mode 100644 api-ref/source/samples/allocations/manage-allocations-request-1.38.json create mode 100644 api-ref/source/samples/allocations/update-allocations-request-1.38.json create mode 100644 api-ref/source/samples/reshaper/post-reshaper-1.38.json create mode 100644 api-ref/source/samples/usages/get-usages-1.38.json create mode 100644 placement/tests/functional/gabbits/consumer-types-1.38.yaml create mode 100644 releasenotes/notes/consumer_type-857b812aef10381e.yaml diff --git a/api-ref/source/allocations.inc b/api-ref/source/allocations.inc index d5f70d53b..9cf8d8f6a 100644 --- a/api-ref/source/allocations.inc +++ b/api-ref/source/allocations.inc @@ -44,6 +44,7 @@ Request - consumer_uuid: consumer_uuid_body - consumer_generation: consumer_generation_min + - consumer_type: consumer_type - project_id: project_id_body - user_id: user_id_body - allocations: allocations_dict_empty @@ -51,9 +52,15 @@ Request - resources: resources - mappings: mappings_in_allocations -Request example (microversions 1.28 - ) +Request example (microversions 1.38 - ) --------------------------------------- +.. literalinclude:: ./samples/allocations/manage-allocations-request-1.38.json + :language: javascript + +Request example (microversions 1.28 - 1.36) +------------------------------------------- + .. literalinclude:: ./samples/allocations/manage-allocations-request-1.28.json :language: javascript @@ -98,12 +105,19 @@ Response - generation: resource_provider_generation - resources: resources - consumer_generation: consumer_generation_get + - consumer_type: consumer_type - project_id: project_id_body_1_12 - user_id: user_id_body_1_12 -Response Example (1.28 - ) +Response Example (1.38 - ) -------------------------- +.. literalinclude:: ./samples/allocations/get-allocations-1.38.json + :language: javascript + +Response Example (1.28 - 1.36) +------------------------------ + .. literalinclude:: ./samples/allocations/get-allocations-1.28.json :language: javascript @@ -146,14 +160,21 @@ Request (microversions 1.12 - ) - allocations: allocations_dict - resources: resources - consumer_generation: consumer_generation_min + - consumer_type: consumer_type - project_id: project_id_body - user_id: user_id_body - generation: resource_provider_generation_optional - mappings: mappings_in_allocations -Request example (microversions 1.28 - ) +Request example (microversions 1.38 - ) --------------------------------------- +.. literalinclude:: ./samples/allocations/update-allocations-request-1.38.json + :language: javascript + +Request example (microversions 1.28 - 1.36) +------------------------------------------- + .. literalinclude:: ./samples/allocations/update-allocations-request-1.28.json :language: javascript diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index 27ddda9fa..232751d81 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -201,6 +201,19 @@ allocation_candidates_same_subtree: specified request group must be an ancestor of the rest. The ``same_subtree`` query parameter can be repeated and each repeat group is treated independently. +consumer_type_req: + type: string + in: query + required: false + min_version: 1.38 + description: | + A string that consists of numbers, ``A-Z``, and ``_`` describing the + consumer type by which to filter usage results. For example, to retrieve + only usage information for 'INSTANCE' type consumers a parameter of + ``consumer_type=INSTANCE`` should be provided. + The ``all`` query parameter may be specified to group all results under + one key, ``all``. The ``unknown`` query parameter may be specified to + group all results under one key, ``unknown``. project_id: &project_id type: string in: query @@ -473,6 +486,13 @@ capacity: required: true description: > The amount of the resource that the provider can accommodate. +consumer_count: + type: integer + in: body + required: true + min_version: 1.38 + description: > + The number of consumers of a particular ``consumer_type``. consumer_generation: &consumer_generation type: integer in: body @@ -489,6 +509,18 @@ consumer_generation_get: consumer_generation_min: <<: *consumer_generation min_version: 1.28 +consumer_type: + type: string + in: body + required: true + min_version: 1.38 + description: > + A string that consists of numbers, ``A-Z``, and ``_`` describing what kind + of consumer is creating, or has created, allocations using a quantity of + inventory. The string is determined by the client when writing allocations + and it is up to the client to ensure correct choices amongst collaborating + services. For example, the compute service may choose to type some + consumers 'INSTANCE' and others 'MIGRATION'. consumer_uuid_body: <<: *consumer_uuid in: body @@ -749,6 +781,12 @@ resources: required: true description: > A dictionary of resource records keyed by resource class name. +resources_single: + type: integer + in: body + required: true + description: > + An amount of resource class consumed in a usage report. step_size: &step_size type: integer in: body diff --git a/api-ref/source/reshaper.inc b/api-ref/source/reshaper.inc index de56948a7..67651f4c0 100644 --- a/api-ref/source/reshaper.inc +++ b/api-ref/source/reshaper.inc @@ -40,6 +40,7 @@ Request - allocations.{consumer_uuid}.user_id: user_id_body - allocations.{consumer_uuid}.mappings: mappings - allocations.{consumer_uuid}.consumer_generation: consumer_generation + - allocations.{consumer_uuid}.consumer_type: consumer_type Request Example --------------- diff --git a/api-ref/source/samples/allocations/get-allocations-1.38.json b/api-ref/source/samples/allocations/get-allocations-1.38.json new file mode 100644 index 000000000..879839602 --- /dev/null +++ b/api-ref/source/samples/allocations/get-allocations-1.38.json @@ -0,0 +1,22 @@ +{ + "allocations": { + "92637880-2d79-43c6-afab-d860886c6391": { + "generation": 2, + "resources": { + "DISK_GB": 5 + } + }, + "ba8e1ef8-7fa3-41a4-9bb4-d7cb2019899b": { + "generation": 8, + "resources": { + "MEMORY_MB": 512, + "VCPU": 2 + } + } + }, + "consumer_generation": 1, + "project_id": "7e67cbf7-7c38-4a32-b85b-0739c690991a", + "user_id": "067f691e-725a-451a-83e2-5c3d13e1dffc", + "consumer_type": "INSTANCE" +} + diff --git a/api-ref/source/samples/allocations/manage-allocations-request-1.38.json b/api-ref/source/samples/allocations/manage-allocations-request-1.38.json new file mode 100644 index 000000000..be537d434 --- /dev/null +++ b/api-ref/source/samples/allocations/manage-allocations-request-1.38.json @@ -0,0 +1,41 @@ +{ + "30328d13-e299-4a93-a102-61e4ccabe474": { + "consumer_generation": 1, + "project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "allocations": { + "e10927c4-8bc9-465d-ac60-d2f79f7e4a00": { + "resources": { + "VCPU": 2, + "MEMORY_MB": 3 + }, + "generation": 4 + } + }, + "consumer_type": "INSTANCE" + }, + "71921e4e-1629-4c5b-bf8d-338d915d2ef3": { + "consumer_generation": 1, + "project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "allocations": {}, + "consumer_type": "MIGRATION" + }, + "48c1d40f-45d8-4947-8d46-52b4e1326df8": { + "consumer_generation": 1, + "project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f", + "allocations": { + "e10927c4-8bc9-465d-ac60-d2f79f7e4a00": { + "resources": { + "VCPU": 4, + "MEMORY_MB": 5 + }, + "generation": 12 + } + }, + "consumer_type": "INSTANCE" + } +} + + diff --git a/api-ref/source/samples/allocations/update-allocations-request-1.38.json b/api-ref/source/samples/allocations/update-allocations-request-1.38.json new file mode 100644 index 000000000..fc6045e2a --- /dev/null +++ b/api-ref/source/samples/allocations/update-allocations-request-1.38.json @@ -0,0 +1,20 @@ +{ + "allocations": { + "4e061c03-611e-4caa-bf26-999dcff4284e": { + "resources": { + "DISK_GB": 20 + } + }, + "89873422-1373-46e5-b467-f0c5e6acf08f": { + "resources": { + "MEMORY_MB": 1024, + "VCPU": 1 + } + } + }, + "consumer_generation": 1, + "user_id": "66cb2f29-c86d-47c3-8af5-69ae7b778c70", + "project_id": "42a32c07-3eeb-4401-9373-68a8cdca6784", + "consumer_type": "INSTANCE" +} + diff --git a/api-ref/source/samples/reshaper/post-reshaper-1.38.json b/api-ref/source/samples/reshaper/post-reshaper-1.38.json new file mode 100644 index 000000000..29453ca11 --- /dev/null +++ b/api-ref/source/samples/reshaper/post-reshaper-1.38.json @@ -0,0 +1,71 @@ +{ + "allocations": { + "9ae60315-80c2-48a0-a168-ca4f27c307e1": { + "allocations": { + "a7466641-cd72-499b-b6c9-c208eacecb3d": { + "resources": { + "DISK_GB": 1000 + } + } + }, + "project_id": "2f0c4ffc-4c4d-407a-b334-56297b871b7f", + "user_id": "cc8a0fe0-2b7c-4392-ae51-747bc73cf473", + "consumer_generation": 1, + "consumer_type": "INSTANCE" + }, + "4a6444e5-10d6-43f6-9a0b-8acce9309ac9": { + "allocations": { + "c4ddddbb-01ee-4814-85c9-f57a962c22ba": { + "resources": { + "VCPU": 1 + } + }, + "a7466641-cd72-499b-b6c9-c208eacecb3d": { + "resources": { + "DISK_GB": 20 + } + } + }, + "project_id": "2f0c4ffc-4c4d-407a-b334-56297b871b7f", + "user_id": "406e1095-71cb-47b9-9b3c-aedb7f663f5a", + "consumer_generation": 1, + "consumer_type": "INSTANCE" + }, + "e10e7ca0-2ac5-4c98-bad9-51c95b1930ed": { + "allocations": { + "c4ddddbb-01ee-4814-85c9-f57a962c22ba": { + "resources": { + "VCPU": 8 + } + } + }, + "project_id": "2f0c4ffc-4c4d-407a-b334-56297b871b7f", + "user_id": "cc8a0fe0-2b7c-4392-ae51-747bc73cf473", + "consumer_generation": 1, + "consumer_type": "INSTANCE" + } + }, + "inventories": { + "c4ddddbb-01ee-4814-85c9-f57a962c22ba": { + "inventories": { + "VCPU": { + "max_unit": 8, + "total": 10 + } + }, + "resource_provider_generation": null + }, + "a7466641-cd72-499b-b6c9-c208eacecb3d": { + "inventories": { + "DISK_GB": { + "min_unit": 10, + "total": 2048, + "max_unit": 1200, + "step_size": 10 + } + }, + "resource_provider_generation": 5 + } + } +} + diff --git a/api-ref/source/samples/usages/get-usages-1.38.json b/api-ref/source/samples/usages/get-usages-1.38.json new file mode 100644 index 000000000..6f2f0ddb6 --- /dev/null +++ b/api-ref/source/samples/usages/get-usages-1.38.json @@ -0,0 +1,22 @@ +{ + "usages" : { + "INSTANCE" : { + "consumer_count" : 5, + "MEMORY_MB" : 512, + "VCPU" : 2, + "DISK_GB" : 5 + }, + "MIGRATION" : { + "DISK_GB" : 5, + "VCPU" : 2, + "consumer_count" : 2, + "MEMORY_MB" : 512 + }, + "unknown" : { + "VCPU" : 2, + "DISK_GB" : 5, + "consumer_count" : 1, + "MEMORY_MB" : 512 + } + } +} diff --git a/api-ref/source/usages.inc b/api-ref/source/usages.inc index 7d320c0ec..2cb348da6 100644 --- a/api-ref/source/usages.inc +++ b/api-ref/source/usages.inc @@ -28,16 +28,32 @@ Request - project_id: project_id - user_id: user_id + - consumer_type: consumer_type_req -Response --------- +Response (microversions 1.38 - ) +-------------------------------- + +.. rest_parameters:: parameters.yaml + + - usages.consumer_type: consumer_type + - usages.consumer_type.consumer_count: consumer_count + - usages.consumer_type.RESOURCE_CLASS: resources_single + +Response Example (microversions 1.38 - ) +---------------------------------------- + +.. literalinclude:: ./samples/usages/get-usages-1.38.json + :language: javascript + +Response (microversions 1.9 - 1.36) +----------------------------------- .. rest_parameters:: parameters.yaml - usages: resources -Response Example ----------------- +Response Example (microversions 1.9 - 1.36) +------------------------------------------- .. literalinclude:: ./samples/usages/get-usages.json :language: javascript diff --git a/placement/handlers/allocation.py b/placement/handlers/allocation.py index bf29eafee..c3120be7d 100644 --- a/placement/handlers/allocation.py +++ b/placement/handlers/allocation.py @@ -22,11 +22,13 @@ from oslo_utils import timeutils from oslo_utils import uuidutils import webob +from placement import db_api from placement import errors from placement import exception from placement.handlers import util as data_util from placement import microversion from placement.objects import allocation as alloc_obj +from placement.objects import consumer_type from placement.objects import resource_provider as rp_obj from placement.policies import allocation as policies from placement.schemas import allocation as schema @@ -55,7 +57,7 @@ def _last_modified_from_allocations(allocations, want_version): return last_modified -def _serialize_allocations_for_consumer(allocations, want_version): +def _serialize_allocations_for_consumer(context, allocations, want_version): """Turn a list of allocations into a dict by resource provider uuid. { @@ -80,6 +82,8 @@ def _serialize_allocations_for_consumer(allocations, want_version): 'user_id': USER_ID, # Generation for consumer >= 1.28 'consumer_generation': 1 + # Consumer Type for consumer >= 1.38 + 'consumer_type': INSTANCE } """ allocation_data = collections.defaultdict(dict) @@ -105,6 +109,18 @@ def _serialize_allocations_for_consumer(allocations, want_version): show_consumer_gen = want_version.matches((1, 28)) if show_consumer_gen: result['consumer_generation'] = consumer.generation + show_consumer_type = want_version.matches((1, 38)) + if show_consumer_type: + # TODO(cdent): This should either access a subclass of + # AttributeCache or the data returned from the persistence layer + # should already have a name. We want to avoid accessing the + # database from the handler layer repeated times. + if consumer.consumer_type_id: + con_type = consumer_type.ConsumerType.get_by_id( + context, consumer.consumer_type_id).name + else: + con_type = consumer_type.DEFAULT_CONSUMER_TYPE + result['consumer_type'] = con_type return result @@ -222,10 +238,11 @@ def inspect_consumers(context, data, want_version): project_id = data[consumer_uuid]['project_id'] user_id = data[consumer_uuid]['user_id'] consumer_generation = data[consumer_uuid].get('consumer_generation') + consumer_type = data[consumer_uuid].get('consumer_type') try: consumer, new_consumer_created = data_util.ensure_consumer( context, consumer_uuid, project_id, user_id, - consumer_generation, want_version) + consumer_generation, consumer_type, want_version) if new_consumer_created: new_consumers_created.append(consumer) consumers[consumer_uuid] = consumer @@ -252,7 +269,8 @@ def list_for_consumer(req): # consumer id. allocations = alloc_obj.get_all_by_consumer_id(context, consumer_id) - output = _serialize_allocations_for_consumer(allocations, want_version) + output = _serialize_allocations_for_consumer( + context, allocations, want_version) last_modified = _last_modified_from_allocations(allocations, want_version) allocations_json = jsonutils.dumps(output) @@ -392,6 +410,21 @@ def _set_allocations_for_consumer(req, schema): } allocation_data = allocations_dict + # NOTE(melwitt): Group all of the database updates in a single transaction + # so that updates get rolled back automatically in the event of a consumer + # generation conflict. + _set_allocations_for_consumer_same_transaction( + context, consumer_uuid, data, allocation_data, want_version) + + req.response.status = 204 + req.response.content_type = None + return req.response + + +@db_api.placement_context_manager.writer +def _set_allocations_for_consumer_same_transaction(context, consumer_uuid, + data, allocation_data, + want_version): allocation_objects = [] # Consumer object saved in case we need to delete the auto-created consumer # record @@ -408,7 +441,8 @@ def _set_allocations_for_consumer(req, schema): # no consumer generation, so this is safe to do. data_util.ensure_consumer( context, consumer_uuid, data.get('project_id'), - data.get('user_id'), data.get('consumer_generation'), want_version) + data.get('user_id'), data.get('consumer_generation'), + data.get('consumer_type'), want_version) allocations = alloc_obj.get_all_by_consumer_id(context, consumer_uuid) for allocation in allocations: allocation.used = 0 @@ -420,7 +454,7 @@ def _set_allocations_for_consumer(req, schema): consumer, created_new_consumer = data_util.ensure_consumer( context, consumer_uuid, data.get('project_id'), data.get('user_id'), data.get('consumer_generation'), - want_version) + data.get('consumer_type'), want_version) for resource_provider_uuid, allocation in allocation_data.items(): resource_provider = rp_objs[resource_provider_uuid] new_allocations = _new_allocations(context, @@ -456,10 +490,6 @@ def _set_allocations_for_consumer(req, schema): 'allocate: %(error)s' % {'error': exc}, comment=errors.CONCURRENT_UPDATE) - req.response.status = 204 - req.response.content_type = None - return req.response - @wsgi_wrapper.PlacementWsgify @microversion.version_handler('1.0', '1.7') @@ -490,12 +520,19 @@ def set_allocations_for_consumer(req): # noqa @wsgi_wrapper.PlacementWsgify # noqa -@microversion.version_handler('1.34') +@microversion.version_handler('1.34', '1.37') @util.require_content('application/json') def set_allocations_for_consumer(req): # noqa return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_34) +@wsgi_wrapper.PlacementWsgify # noqa +@microversion.version_handler('1.38') +@util.require_content('application/json') +def set_allocations_for_consumer(req): # noqa + return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_38) + + @wsgi_wrapper.PlacementWsgify @microversion.version_handler('1.13') @util.require_content('application/json') @@ -508,6 +545,8 @@ def set_allocations(req): want_schema = schema.POST_ALLOCATIONS_V1_28 if want_version.matches((1, 34)): want_schema = schema.POST_ALLOCATIONS_V1_34 + if want_version.matches((1, 38)): + want_schema = schema.POST_ALLOCATIONS_V1_38 data = util.extract_json(req.body, want_schema) consumers, new_consumers_created = inspect_consumers( diff --git a/placement/handlers/reshaper.py b/placement/handlers/reshaper.py index ef70e4034..fd0369dc9 100644 --- a/placement/handlers/reshaper.py +++ b/placement/handlers/reshaper.py @@ -46,7 +46,9 @@ def reshape(req): context.can(policies.RESHAPE) reshaper_schema = schema.POST_RESHAPER_SCHEMA - if want_version.matches((1, 34)): + if want_version.matches((1, 38)): + reshaper_schema = schema.POST_RESHAPER_SCHEMA_V1_38 + elif want_version.matches((1, 34)): reshaper_schema = schema.POST_RESHAPER_SCHEMA_V1_34 data = util.extract_json(req.body, reshaper_schema) inventories = data['inventories'] diff --git a/placement/handlers/usage.py b/placement/handlers/usage.py index dff126acf..fe82be0c7 100644 --- a/placement/handlers/usage.py +++ b/placement/handlers/usage.py @@ -11,6 +11,8 @@ # under the License. """Placement API handlers for usage information.""" +import collections + from oslo_serialization import jsonutils from oslo_utils import encodeutils from oslo_utils import timeutils @@ -90,6 +92,7 @@ def get_total_usages(req): """ project_id = req.GET.get('project_id') user_id = req.GET.get('user_id') + consumer_type = req.GET.get('consumer_type') context = req.environ['placement.context'] context.can( @@ -97,14 +100,36 @@ def get_total_usages(req): target={'project_id': project_id}) want_version = req.environ[microversion.MICROVERSION_ENVIRON] - util.validate_query_params(req, schema.GET_USAGES_SCHEMA_1_9) + want_schema = schema.GET_USAGES_SCHEMA_1_9 + show_consumer_type = want_version.matches((1, 38)) + if show_consumer_type: + want_schema = schema.GET_USAGES_SCHEMA_V1_38 + util.validate_query_params(req, want_schema) - usages = usage_obj.get_all_by_project_user(context, project_id, - user_id=user_id) + if show_consumer_type: + usages = usage_obj.get_by_consumer_type( + context, project_id, user_id=user_id, consumer_type=consumer_type) + else: + usages = usage_obj.get_all_by_project_user(context, project_id, + user_id=user_id) response = req.response - usages_dict = {'usages': {resource.resource_class: resource.usage - for resource in usages}} + if show_consumer_type: + usage = collections.defaultdict(dict) + for resource in usages: + ct = resource.consumer_type + rc = resource.resource_class + cc = resource.consumer_count + used = resource.usage + usage[ct][rc] = used + usage[ct]['consumer_count'] = cc + usages_dict = { + 'usages': usage + } + else: + usages_dict = {'usages': {resource.resource_class: resource.usage + for resource in usages}} + response.body = encodeutils.to_utf8(jsonutils.dumps(usages_dict)) req.response.content_type = 'application/json' if want_version.matches((1, 15)): diff --git a/placement/handlers/util.py b/placement/handlers/util.py index 37919e6c9..c37e9ba09 100644 --- a/placement/handlers/util.py +++ b/placement/handlers/util.py @@ -17,6 +17,7 @@ import webob from placement import errors from placement import exception from placement.objects import consumer as consumer_obj +from placement.objects import consumer_type as consumer_type_obj from placement.objects import project as project_obj from placement.objects import user as user_obj @@ -24,8 +25,78 @@ from placement.objects import user as user_obj LOG = logging.getLogger(__name__) +def fetch_consumer_type_id(ctx, name): + """Tries to fetch the provided consumer_type and creates a new one if it + does not exist. + + :param ctx: The request context. + :param name: The name of the consumer type. + :returns: The id of the ConsumerType object. + """ + try: + cons_type = consumer_type_obj.ConsumerType.get_by_name(ctx, name) + except exception.ConsumerTypeNotFound: + cons_type = consumer_type_obj.ConsumerType(ctx, name=name) + try: + cons_type.create() + except exception.ConsumerTypeExists: + # another thread created concurrently, so try again + return fetch_consumer_type_id(ctx, name) + return cons_type.id + + +def _get_or_create_project(ctx, project_id): + try: + proj = project_obj.Project.get_by_external_id(ctx, project_id) + except exception.NotFound: + # Auto-create the project if we found no record of it... + try: + proj = project_obj.Project(ctx, external_id=project_id) + proj.create() + except exception.ProjectExists: + # No worries, another thread created this project already + proj = project_obj.Project.get_by_external_id(ctx, project_id) + return proj + + +def _get_or_create_user(ctx, user_id): + try: + user = user_obj.User.get_by_external_id(ctx, user_id) + except exception.NotFound: + # Auto-create the user if we found no record of it... + try: + user = user_obj.User(ctx, external_id=user_id) + user.create() + except exception.UserExists: + # No worries, another thread created this user already + user = user_obj.User.get_by_external_id(ctx, user_id) + return user + + +def _create_consumer(ctx, consumer_uuid, project, user, consumer_type_id): + created_new_consumer = False + try: + consumer = consumer_obj.Consumer( + ctx, uuid=consumer_uuid, project=project, user=user, + consumer_type_id=consumer_type_id) + consumer.create() + created_new_consumer = True + except exception.ConsumerExists: + # Another thread created this consumer already, verify whether + # the consumer type matches + consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) + # If the types don't match, update the consumer record + if consumer_type_id != consumer.consumer_type_id: + LOG.debug("Supplied consumer type for consumer %s was " + "different than existing record. Updating " + "consumer record.", consumer_uuid) + consumer.consumer_type_id = consumer_type_id + consumer.update() + return consumer, created_new_consumer + + def ensure_consumer(ctx, consumer_uuid, project_id, user_id, - consumer_generation, want_version): + consumer_generation, consumer_type, want_version): """Ensures there are records in the consumers, projects and users table for the supplied external identifiers. @@ -44,35 +115,19 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id, :param user_id: The external ID of the user consuming the resources. :param consumer_generation: The generation provided by the user for this consumer. + :param consumer_type: The type of consumer provided by the user. :param want_version: the microversion matcher. :raises webob.exc.HTTPConflict if consumer generation is required and there was a mismatch """ created_new_consumer = False requires_consumer_generation = want_version.matches((1, 28)) + requires_consumer_type = want_version.matches((1, 38)) if project_id is None: project_id = ctx.config.placement.incomplete_consumer_project_id user_id = ctx.config.placement.incomplete_consumer_user_id - try: - proj = project_obj.Project.get_by_external_id(ctx, project_id) - except exception.NotFound: - # Auto-create the project if we found no record of it... - try: - proj = project_obj.Project(ctx, external_id=project_id) - proj.create() - except exception.ProjectExists: - # No worries, another thread created this project already - proj = project_obj.Project.get_by_external_id(ctx, project_id) - try: - user = user_obj.User.get_by_external_id(ctx, user_id) - except exception.NotFound: - # Auto-create the user if we found no record of it... - try: - user = user_obj.User(ctx, external_id=user_id) - user.create() - except exception.UserExists: - # No worries, another thread created this user already - user = user_obj.User.get_by_external_id(ctx, user_id) + proj = _get_or_create_project(ctx, project_id) + user = _get_or_create_user(ctx, user_id) try: consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) @@ -101,14 +156,14 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id, # existing consumer's record. If the eventual call to # AllocationList.replace_all() fails for whatever reason (say, a # resource provider generation conflict or out of resources failure), - # we will end up deleting the auto-created consumer but we MAY not undo - # the changes to the second consumer's project and user ID. I say MAY - # and not WILL NOT because I'm not sure that the exception that gets - # raised from AllocationList.replace_all() will cause the context - # manager's transaction to rollback automatically. I believe that the - # same transaction context is used for both util.ensure_consumer() and - # AllocationList.replace_all() within the same HTTP request, but need - # to test this to be 100% certain... + # we will end up deleting the auto-created consumer and we will undo + # the changes to the second consumer's project and user ID. + # NOTE(melwitt): The aforementioned rollback of changes is predicated + # on the fact that the same transaction context is used for both + # util.ensure_consumer() and AllocationList.replace_all() within the + # same HTTP request. The @db_api.placement_context_manager.writer + # decorator on the outermost method will nest to methods called within + # the outermost method. if (project_id != consumer.project.external_id or user_id != consumer.user.external_id): LOG.debug("Supplied project or user ID for consumer %s was " @@ -117,6 +172,15 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id, consumer.project = proj consumer.user = user consumer.update() + # Update the consumer type if it's different than the existing one. + if requires_consumer_type: + cons_type_id = fetch_consumer_type_id(ctx, consumer_type) + if cons_type_id != consumer.consumer_type_id: + LOG.debug("Supplied consumer type for consumer %s was " + "different than existing record. Updating " + "consumer record.", consumer_uuid) + consumer.consumer_type_id = cons_type_id + consumer.update() except exception.NotFound: # If we are attempting to modify or create allocations after 1.26, we # need a consumer generation specified. The user must have specified @@ -129,14 +193,11 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id, 'consumer generation conflict - ' 'expected null but got %s' % consumer_generation, comment=errors.CONCURRENT_UPDATE) + cons_type_id = (fetch_consumer_type_id(ctx, consumer_type) + if requires_consumer_type else None) # No such consumer. This is common for new allocations. Create the # consumer record - try: - consumer = consumer_obj.Consumer( - ctx, uuid=consumer_uuid, project=proj, user=user) - consumer.create() - created_new_consumer = True - except exception.ConsumerExists: - # No worries, another thread created this user already - consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) + consumer, created_new_consumer = _create_consumer( + ctx, consumer_uuid, proj, user, cons_type_id) + return consumer, created_new_consumer diff --git a/placement/microversion.py b/placement/microversion.py index a83e628de..b8d31abbf 100644 --- a/placement/microversion.py +++ b/placement/microversion.py @@ -89,6 +89,16 @@ VERSIONS = [ '1.36', # Add a `same_subtree` parameter on GET /allocation_candidates # and allow resourceless requests for groups in `same_subtree`. '1.37', # Allow re-parenting and un-parenting resource providers + '1.38', # Adds ``consumer_type`` (required) key in the request body of + # ``POST /allocations``, ``PUT /allocations/{consumer_uuid}`` + # and in the response of ``GET /allocations/{consumer_uuid}``. + # ``GET /usages`` request will also gain ``consumer_type`` key as + # an optional queryparam to filter usages based on consumer_types. + # ``GET /usages`` response will group results based on the + # consumer type and will include a new ``consumer_count`` key per + # type irrespective of whether the ``consumer_type`` was specified + # in the request. The corresponding changes to ``/reshaper`` are + # included. ] diff --git a/placement/objects/usage.py b/placement/objects/usage.py index 1e1824490..02d08cff6 100644 --- a/placement/objects/usage.py +++ b/placement/objects/usage.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import sqlalchemy as sa from sqlalchemy import distinct from sqlalchemy import func from sqlalchemy import sql @@ -87,12 +88,15 @@ def _get_all_by_project_user(context, project_id, user_id=None, query = query.filter(models.User.external_id == user_id) query = query.group_by(models.Allocation.resource_class_id) - if consumer_type: - # NOTE(melwitt): We have to count separately in order to get a count of - # unique consumers. If we count after grouping by resource class, we - # will count duplicate consumers for any unique consumer that consumes - # more than one resource class simultaneously (example: an instance - # consuming both VCPU and MEMORY_MB). + # Handle 'all' or 'unknown' consumer type. The consumer_type is True if + # 'all' is requested, and None if 'unknown' is requested. + if consumer_type is None or consumer_type: + # NOTE(melwitt): We have to count the number of consumers in a separate + # query in order to get a count of unique consumers. If we count in the + # same query after grouping by resource class, we will count duplicate + # consumers for any unique consumer that consumes more than one + # resource class simultaneously (example: an instance consuming both + # VCPU and MEMORY_MB). count_query = (context.session.query( func.count(distinct(models.Allocation.consumer_id))) .join(models.Consumer, @@ -105,12 +109,19 @@ def _get_all_by_project_user(context, project_id, user_id=None, models.User, models.Consumer.user_id == models.User.id) count_query = count_query.filter( models.User.external_id == user_id) - unique_consumer_count = count_query.scalar() + if consumer_type is None: + count_query = count_query.filter( + models.Consumer.consumer_type_id == sa.null()) + number_of_unique_consumers = count_query.scalar() + + # Filter for unknown consumer type if specified. + if consumer_type is None: + query = query.filter(models.Consumer.consumer_type_id == sa.null()) result = [dict(resource_class=context.rc_cache.string_from_id(item[0]), usage=item[1], - consumer_type="all", - consumer_count=unique_consumer_count) + consumer_type='all' if consumer_type else 'unknown', + consumer_count=number_of_unique_consumers) for item in query.all()] else: result = [dict(resource_class=context.rc_cache.string_from_id(item[0]), @@ -123,9 +134,11 @@ def _get_all_by_project_user(context, project_id, user_id=None, @db_api.placement_context_manager.reader def _get_by_consumer_type(context, project_id, user_id=None, consumer_type=None): - if consumer_type == 'all': + if consumer_type in ('all', 'unknown'): + cons_type = True if consumer_type == 'all' else None return _get_all_by_project_user(context, project_id, user_id, - consumer_type=True) + consumer_type=cons_type) + query = (context.session.query( models.Allocation.resource_class_id, func.coalesce(func.sum(models.Allocation.used), 0), diff --git a/placement/rest_api_version_history.rst b/placement/rest_api_version_history.rst index 04fef40e8..7c5e766ad 100644 --- a/placement/rest_api_version_history.rst +++ b/placement/rest_api_version_history.rst @@ -677,4 +677,24 @@ Add support for re-parenting and un-parenting a resource provider via ``PUT /resource_providers/{uuid}`` API by allowing changing the ``parent_provider_uuid`` to any existing provider, except providers in same subtree. Un-parenting can be achieved by setting the ``parent_provider_uuid`` -to ``null``. This means that the provider becomes a new root provider. \ No newline at end of file +to ``null``. This means that the provider becomes a new root provider. + +1.38 - Support consumer_type in allocations, usage and reshaper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: Xena + +Adds support for a ``consumer_type`` (required) key in the request body of +``POST /allocations``, ``PUT /allocations/{consumer_uuid}`` and in the response +of ``GET /allocations/{consumer_uuid}``. ``GET /usages`` requests gain a +``consumer_type`` key as an optional query parameter to filter usages based on +consumer_types. ``GET /usages`` response will group results based on the +consumer type and will include a new ``consumer_count`` key per type +irrespective of whether the ``consumer_type`` was specified in the request. If +an ``all`` ``consumer_type`` key is provided, all results are grouped under one +key, ``all``. Older allocations which were not created with a consumer type are +considered to have an ``unknown`` ``consumer_type``. If an ``unknown`` +``consumer_type`` key is provided, all results are grouped under one key, +``unknown``. + +The corresponding changes to ``POST /reshaper`` are included. diff --git a/placement/schemas/allocation.py b/placement/schemas/allocation.py index 414372f9e..bdf93795f 100644 --- a/placement/schemas/allocation.py +++ b/placement/schemas/allocation.py @@ -190,3 +190,18 @@ POST_ALLOCATIONS_V1_34 = copy.deepcopy(POST_ALLOCATIONS_V1_28) POST_ALLOCATIONS_V1_34["patternProperties"] = { common.UUID_PATTERN: ALLOCATION_SCHEMA_V1_34 } + +# A required consumer type was added to the allocations dicts in this +# version of PUT /allocations/{consumer_uuid} and POST /allocations. +ALLOCATION_SCHEMA_V1_38 = copy.deepcopy(ALLOCATION_SCHEMA_V1_34) +ALLOCATION_SCHEMA_V1_38['properties']['consumer_type'] = { + "type": "string", + "pattern": common.CONSUMER_TYPE_PATTERN, + "minLength": 1, + "maxLength": 255, +} +ALLOCATION_SCHEMA_V1_38['required'].append("consumer_type") +POST_ALLOCATIONS_V1_38 = copy.deepcopy(POST_ALLOCATIONS_V1_34) +POST_ALLOCATIONS_V1_38["patternProperties"] = { + common.UUID_PATTERN: ALLOCATION_SCHEMA_V1_38 +} diff --git a/placement/schemas/common.py b/placement/schemas/common.py index da1d691ea..d55da1bd1 100644 --- a/placement/schemas/common.py +++ b/placement/schemas/common.py @@ -20,6 +20,8 @@ RC_PATTERN = _RC_TRAIT_PATTERN _CUSTOM_RC_TRAIT_PATTERN = "^CUSTOM_%s+$" % _RC_TRAIT_CHAR CUSTOM_RC_PATTERN = _CUSTOM_RC_TRAIT_PATTERN CUSTOM_TRAIT_PATTERN = _CUSTOM_RC_TRAIT_PATTERN +CONSUMER_TYPE_PATTERN = _RC_TRAIT_PATTERN +CONSUMER_TYPE_GET_PATTERN = "%s|^all|^unknown$" % CONSUMER_TYPE_PATTERN # The suffix used with request groups. Prior to 1.33, the group were numbered. # With 1.33 they become alphanumeric, '_', and '-' with a length limit of 64. diff --git a/placement/schemas/reshaper.py b/placement/schemas/reshaper.py index 3da8a812b..ce881e500 100644 --- a/placement/schemas/reshaper.py +++ b/placement/schemas/reshaper.py @@ -50,3 +50,8 @@ POST_RESHAPER_SCHEMA_V1_34 = copy.deepcopy(POST_RESHAPER_SCHEMA) ALLOCATIONS_V1_34 = copy.deepcopy(allocation.POST_ALLOCATIONS_V1_34) ALLOCATIONS_V1_34['minProperties'] = 0 POST_RESHAPER_SCHEMA_V1_34['properties']['allocations'] = ALLOCATIONS_V1_34 + +POST_RESHAPER_SCHEMA_V1_38 = copy.deepcopy(POST_RESHAPER_SCHEMA_V1_34) +ALLOCATIONS_V1_38 = copy.deepcopy(allocation.POST_ALLOCATIONS_V1_38) +ALLOCATIONS_V1_38['minProperties'] = 0 +POST_RESHAPER_SCHEMA_V1_38['properties']['allocations'] = ALLOCATIONS_V1_38 diff --git a/placement/schemas/usage.py b/placement/schemas/usage.py index 847e745fa..783d9d94a 100644 --- a/placement/schemas/usage.py +++ b/placement/schemas/usage.py @@ -11,6 +11,11 @@ # under the License. """Placement API schemas for usage information.""" +import copy + +from placement.schemas import common + + # Represents the allowed query string parameters to GET /usages GET_USAGES_SCHEMA_1_9 = { "type": "object", @@ -31,3 +36,15 @@ GET_USAGES_SCHEMA_1_9 = { ], "additionalProperties": False, } + + +# An optional consumer type was added to the usage dicts in this +# version of GET /usages. + +GET_USAGES_SCHEMA_V1_38 = copy.deepcopy(GET_USAGES_SCHEMA_1_9) +GET_USAGES_SCHEMA_V1_38['properties']['consumer_type'] = { + "type": "string", + "pattern": common.CONSUMER_TYPE_GET_PATTERN, + "minLength": 1, + "maxLength": 255, +} diff --git a/placement/tests/functional/gabbits/allocations-legacy-rbac.yaml b/placement/tests/functional/gabbits/allocations-legacy-rbac.yaml index 95dfe84fb..c27ef0aa8 100644 --- a/placement/tests/functional/gabbits/allocations-legacy-rbac.yaml +++ b/placement/tests/functional/gabbits/allocations-legacy-rbac.yaml @@ -11,14 +11,18 @@ vars: x-roles: admin,member,reader accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.36 here because 1.37 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.36 openstack-system-scope: all - &system_reader_headers x-auth-token: user x-roles: reader accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.36 here because 1.37 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.36 openstack-system-scope: all - &project_admin_headers x-auth-token: user @@ -26,21 +30,27 @@ vars: x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.36 here because 1.37 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.36 - &project_member_headers x-auth-token: user x-roles: member,reader x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.36 here because 1.37 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.36 - &project_reader_headers x-auth-token: user x-roles: reader x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.36 here because 1.37 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.36 - &agg_1 f918801a-5e54-4bee-9095-09a9d0c786b8 - &agg_2 a893eb5c-e2a0-4251-ab26-f71d3b0cfc0b diff --git a/placement/tests/functional/gabbits/allocations-policy.yaml b/placement/tests/functional/gabbits/allocations-policy.yaml index ebee30cb6..9d43c12b3 100644 --- a/placement/tests/functional/gabbits/allocations-policy.yaml +++ b/placement/tests/functional/gabbits/allocations-policy.yaml @@ -10,7 +10,9 @@ defaults: x-auth-token: user accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 tests: diff --git a/placement/tests/functional/gabbits/allocations-secure-rbac.yaml b/placement/tests/functional/gabbits/allocations-secure-rbac.yaml index 5d6908d23..0f0f3db2f 100644 --- a/placement/tests/functional/gabbits/allocations-secure-rbac.yaml +++ b/placement/tests/functional/gabbits/allocations-secure-rbac.yaml @@ -11,14 +11,18 @@ vars: x-roles: admin,member,reader accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 openstack-system-scope: all - &system_reader_headers x-auth-token: user x-roles: reader accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 openstack-system-scope: all - &project_admin_headers x-auth-token: user @@ -26,21 +30,27 @@ vars: x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 - &project_member_headers x-auth-token: user x-roles: member,reader x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 - &project_reader_headers x-auth-token: user x-roles: reader x-project-id: *project_id accept: application/json content-type: application/json - openstack-api-version: placement latest + # We need 1.37 here because 1.38 required consumer_type which these + # allocations do not have. + openstack-api-version: placement 1.37 - &agg_1 f918801a-5e54-4bee-9095-09a9d0c786b8 - &agg_2 a893eb5c-e2a0-4251-ab26-f71d3b0cfc0b diff --git a/placement/tests/functional/gabbits/consumer-types-1.38.yaml b/placement/tests/functional/gabbits/consumer-types-1.38.yaml new file mode 100644 index 000000000..72db6dfa8 --- /dev/null +++ b/placement/tests/functional/gabbits/consumer-types-1.38.yaml @@ -0,0 +1,265 @@ +# Test consumer types work as designed. +fixtures: + - AllocationFixture + +defaults: + request_headers: + x-auth-token: admin + accept: application/json + content-type: application/json + openstack-api-version: placement 1.38 + +tests: + +- name: 400 on no consumer type post + POST: /allocations + data: + f5a91a0a-e111-4a9c-8a33-7b320ae1e52a: + consumer_generation: null + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 400 + response_strings: + - "'consumer_type' is a required property" + +- name: 400 on no consumer type put + PUT: /allocations/f5a91a0a-e111-4a9c-8a33-7b320ae1e52a + data: + consumer_generation: null + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 400 + response_strings: + - "'consumer_type' is a required property" + +- name: consumer type post + POST: /allocations + data: + f5a91a0a-e111-4a9c-8a33-7b320ae1e52a: + consumer_type: INSTANCE + consumer_generation: null + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 204 + +- name: consumer type put + PUT: /allocations/f5a91a0a-e111-4a9c-8a33-7b320ae1e52a + data: + consumer_generation: 1 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_type: PONY + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 204 + +- name: consumer put without type + PUT: /allocations/4fa4553e-e739-4f0b-a758-2fa79fda2ee0 + request_headers: + openstack-api-version: placement 1.36 + data: + consumer_generation: null + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 204 + +- name: reset to new type + PUT: /allocations/4fa4553e-e739-4f0b-a758-2fa79fda2ee0 + data: + consumer_generation: 1 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_type: INSTANCE + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 204 + +- name: malformed consumer type put + PUT: /allocations/4fa4553e-e739-4f0b-a758-2fa79fda2ee0 + data: + consumer_generation: 1 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_type: instance + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 400 + response_strings: + - "'instance' does not match '^[A-Z0-9_]+$'" + +- name: malformed consumer type post + POST: /allocations + data: + 4fa4553e-e739-4f0b-a758-2fa79fda2ee0: + consumer_generation: 1 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_type: instance + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + status: 400 + response_strings: + - "'instance' does not match '^[A-Z0-9_]+$'" + +# check usages, some allocations are pre-provided by the fixture +- name: usages include consumer_type + GET: /usages?project_id=$ENVIRON['PROJECT_ID'] + response_json_paths: + $.usages.PONY: + consumer_count: 1 + DISK_GB: 10 + $.usages.INSTANCE: + consumer_count: 1 + DISK_GB: 10 + $.usages.unknown: + consumer_count: 3 + DISK_GB: 1020 + VCPU: 7 + +- name: limit usages by consumer_type + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=PONY + response_json_paths: + $.usages.`len`: 1 + $.usages.PONY: + consumer_count: 1 + DISK_GB: 10 + +- name: limit usages bad consumer_type + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=COW + response_json_paths: + $.usages.`len`: 0 + +- name: limit usages by all + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=all + response_json_paths: + $.usages.`len`: 1 + $.usages.all: + consumer_count: 5 + DISK_GB: 1040 + VCPU: 7 + +- name: ALL is not all + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=ALL + response_json_paths: + $.usages.`len`: 0 + +- name: limit usages by unknown + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=unknown + response_json_paths: + $.usages.`len`: 1 + $.usages.unknown: + consumer_count: 3 + DISK_GB: 1020 + VCPU: 7 + +- name: UNKNOWN is not unknown + GET: /usages?project_id=$ENVIRON['PROJECT_ID']&consumer_type=UNKNOWN + response_json_paths: + $.usages.`len`: 0 + +- name: reshaper accepts consumer type + POST: /reshaper + data: + inventories: + $ENVIRON['RP_UUID']: + # It's 9 because of the previous work + resource_provider_generation: 9 + inventories: + DISK_GB: + total: 2048 + VCPU: + total: 97 + allocations: + 4b01cd5a-9e12-46d7-9b2a-5bc0f6040a40: + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_generation: null + consumer_type: RESHAPED + status: 204 + +- name: confirm reshaped allocations + GET: /allocations/4b01cd5a-9e12-46d7-9b2a-5bc0f6040a40 + response_json_paths: + $.consumer_type: RESHAPED + +- name: reshaper requires consumer type + POST: /reshaper + data: + inventories: + $ENVIRON['RP_UUID']: + # It's 9 because of the previous work + resource_provider_generation: 9 + inventories: + DISK_GB: + total: 2048 + VCPU: + total: 97 + allocations: + 4b01cd5a-9e12-46d7-9b2a-5bc0f6040a40: + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_generation: 1 + status: 400 + response_strings: + - "'consumer_type' is a required" + +- name: reshaper refuses consumer type earlier microversion + request_headers: + openstack-api-version: placement 1.36 + POST: /reshaper + data: + inventories: + $ENVIRON['RP_UUID']: + # It's 9 because of the previous work + resource_provider_generation: 9 + inventories: + DISK_GB: + total: 2048 + VCPU: + total: 97 + allocations: + 4b01cd5a-9e12-46d7-9b2a-5bc0f6040a40: + allocations: + $ENVIRON['RP_UUID']: + resources: + DISK_GB: 10 + project_id: $ENVIRON['PROJECT_ID'] + user_id: $ENVIRON['USER_ID'] + consumer_generation: 1 + consumer_type: RESHAPED + status: 400 + response_strings: + - "JSON does not validate: Additional properties are not allowed" + - "'consumer_type' was unexpected" diff --git a/placement/tests/functional/gabbits/microversion.yaml b/placement/tests/functional/gabbits/microversion.yaml index f3d500766..8fbc54987 100644 --- a/placement/tests/functional/gabbits/microversion.yaml +++ b/placement/tests/functional/gabbits/microversion.yaml @@ -41,13 +41,13 @@ tests: response_json_paths: $.errors[0].title: Not Acceptable -- name: latest microversion is 1.37 +- name: latest microversion is 1.38 GET: / request_headers: openstack-api-version: placement latest response_headers: vary: /openstack-api-version/ - openstack-api-version: placement 1.37 + openstack-api-version: placement 1.38 - name: other accept header bad version GET: / diff --git a/placement/tests/unit/handlers/test_util.py b/placement/tests/unit/handlers/test_util.py index 82bb91896..96f7e5265 100644 --- a/placement/tests/unit/handlers/test_util.py +++ b/placement/tests/unit/handlers/test_util.py @@ -25,6 +25,7 @@ from placement import exception from placement.handlers import util from placement import microversion from placement.objects import consumer as consumer_obj +from placement.objects import consumer_type as consumer_type_obj from placement.objects import project as project_obj from placement.objects import user as user_obj from placement.tests.unit import base @@ -54,6 +55,12 @@ class TestEnsureConsumer(base.ContextTestCase): self.mock_consumer_create = self.useFixture(fixtures.MockPatch( 'placement.objects.consumer.' 'Consumer.create')).mock + self.mock_consumer_update = self.useFixture(fixtures.MockPatch( + 'placement.objects.consumer.' + 'Consumer.update')).mock + self.mock_consumer_type_get = self.useFixture(fixtures.MockPatch( + 'placement.objects.consumer_type.' + 'ConsumerType.get_by_name')).mock self.ctx = context.RequestContext(user_id='fake', project_id='fake') self.ctx.config = self.conf self.consumer_id = uuidsentinel.consumer @@ -71,6 +78,12 @@ class TestEnsureConsumer(base.ContextTestCase): mv_parsed.min_version = microversion_parse.parse_version_string( microversion.min_version_string()) self.after_version = mv_parsed + mv_parsed = microversion_parse.Version(1, 38) + mv_parsed.max_version = microversion_parse.parse_version_string( + microversion.max_version_string()) + mv_parsed.min_version = microversion_parse.parse_version_string( + microversion.min_version_string()) + self.cons_type_req_version = mv_parsed def test_no_existing_project_user_consumer_before_gen_success(self): """Tests that we don't require a consumer_generation=None before the @@ -83,7 +96,7 @@ class TestEnsureConsumer(base.ContextTestCase): consumer_gen = 1 # should be ignored util.ensure_consumer( self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.before_version) + consumer_gen, 'TYPE', self.before_version) self.mock_project_get.assert_called_once_with( self.ctx, self.project_id) @@ -106,7 +119,7 @@ class TestEnsureConsumer(base.ContextTestCase): consumer_gen = None # should NOT be ignored (and None is expected) util.ensure_consumer( self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.after_version) + consumer_gen, 'TYPE', self.after_version) self.mock_project_get.assert_called_once_with( self.ctx, self.project_id) @@ -131,7 +144,7 @@ class TestEnsureConsumer(base.ContextTestCase): webob.exc.HTTPConflict, util.ensure_consumer, self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.after_version) + consumer_gen, 'TYPE', self.after_version) def test_no_existing_project_user_consumer_use_incomplete(self): """Verify that if the project_id arg is None, that we fall back to the @@ -144,7 +157,7 @@ class TestEnsureConsumer(base.ContextTestCase): consumer_gen = None # should NOT be ignored (and None is expected) util.ensure_consumer( self.ctx, self.consumer_id, None, None, - consumer_gen, self.before_version) + consumer_gen, 'TYPE', self.before_version) self.mock_project_get.assert_called_once_with( self.ctx, self.conf.placement.incomplete_consumer_project_id) @@ -170,7 +183,7 @@ class TestEnsureConsumer(base.ContextTestCase): consumer_gen = None # should be ignored util.ensure_consumer( self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.before_version) + consumer_gen, 'TYPE', self.before_version) self.mock_project_create.assert_not_called() self.mock_user_create.assert_not_called() @@ -192,7 +205,7 @@ class TestEnsureConsumer(base.ContextTestCase): consumer_gen = 2 # should NOT be ignored (and 2 is expected) util.ensure_consumer( self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.after_version) + consumer_gen, 'TYPE', self.after_version) self.mock_project_create.assert_not_called() self.mock_user_create.assert_not_called() @@ -217,4 +230,62 @@ class TestEnsureConsumer(base.ContextTestCase): webob.exc.HTTPConflict, util.ensure_consumer, self.ctx, self.consumer_id, self.project_id, self.user_id, - consumer_gen, self.after_version) + consumer_gen, 'TYPE', self.after_version) + + def test_existing_consumer_different_consumer_type_supplied(self): + """Tests that we update a consumer's type ID if the one supplied by the + user is different than the one in the existing record. + """ + proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id) + self.mock_project_get.return_value = proj + user = user_obj.User(self.ctx, id=1, external_id=self.user_id) + self.mock_user_get.return_value = user + # Consumer currently has type ID = 1 + consumer = consumer_obj.Consumer( + self.ctx, id=1, project=proj, user=user, generation=1, + consumer_type_id=1) + self.mock_consumer_get.return_value = consumer + # Supplied consumer type ID = 2 + consumer_type = consumer_type_obj.ConsumerType( + self.ctx, id=2, name='TYPE') + self.mock_consumer_type_get.return_value = consumer_type + + consumer_gen = 1 + util.ensure_consumer( + self.ctx, self.consumer_id, self.project_id, self.user_id, + consumer_gen, 'TYPE', self.cons_type_req_version) + # Expect 1 call to update() to update to the supplied consumer type ID + self.mock_consumer_update.assert_called_once_with() + # Consumer should have the new consumer type ID = 2 + self.assertEqual(2, consumer.consumer_type_id) + + def test_consumer_create_exists_different_consumer_type_supplied(self): + """Tests that we update a consumer's type ID if the one supplied by a + racing request is different than the one in the existing (recently + created) record. + """ + proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id) + self.mock_project_get.return_value = proj + user = user_obj.User(self.ctx, id=1, external_id=self.user_id) + self.mock_user_get.return_value = user + # Request A recently created consumer has type ID = 1 + consumer = consumer_obj.Consumer( + self.ctx, id=1, project=proj, user=user, generation=1, + consumer_type_id=1, uuid=uuidsentinel.consumer) + self.mock_consumer_get.return_value = consumer + # Request B supplied consumer type ID = 2 + consumer_type = consumer_type_obj.ConsumerType( + self.ctx, id=2, name='TYPE') + self.mock_consumer_type_get.return_value = consumer_type + # Request B will encounter ConsumerExists as Request A just created it + self.mock_consumer_create.side_effect = ( + exception.ConsumerExists(uuid=uuidsentinel.consumer)) + + consumer_gen = 1 + util.ensure_consumer( + self.ctx, self.consumer_id, self.project_id, self.user_id, + consumer_gen, 'TYPE', self.cons_type_req_version) + # Expect 1 call to update() to update to the supplied consumer type ID + self.mock_consumer_update.assert_called_once_with() + # Consumer should have the new consumer type ID = 2 + self.assertEqual(2, consumer.consumer_type_id) diff --git a/releasenotes/notes/consumer_type-857b812aef10381e.yaml b/releasenotes/notes/consumer_type-857b812aef10381e.yaml new file mode 100644 index 000000000..f8ddab926 --- /dev/null +++ b/releasenotes/notes/consumer_type-857b812aef10381e.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + Microversion 1.38 adds support for a ``consumer_type`` (required) key in + the request body of ``POST /allocations``, ``PUT + /allocations/{consumer_uuid}`` and in the response of ``GET + /allocations/{consumer_uuid}``. ``GET /usages`` requests gain a + ``consumer_type`` key as an optional query parameter to filter usages based + on consumer_types. ``GET /usages`` response will group results based on the + consumer type and will include a new ``consumer_count`` key per type + irrespective of whether the ``consumer_type`` was specified in the request. + If an ``all`` ``consumer_type`` key is provided, all results are grouped + under one key, ``all``. Older allocations which were not created with a + consumer type are considered to have an ``unknown`` ``consumer_type``. If + an ``unknown`` ``consumer_type`` key is provided, all results are grouped + under one key, ``unknown``. + + The corresponding changes to ``POST /reshaper`` are included.