openstack/nova
Code Issues Proposed changes

Add a microversion for consumer generation support

Browse Source 
This patch adds new placement API microversion for handling consumer
generations.

Change-Id: I978fdea51f2d6c2572498ef80640c92ab38afe65
Co-Authored-By: Ed Leafe <ed@leafe.com>
Blueprint: add-consumer-generation
This commit is contained in:
Jay Pipes 2018-05-01 18:38:15 -04:00 committed by Chris Dent
parent b992b90d73
commit 092820939d
19 changed files with 761 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nova/api/openstack/placement/errors.py
View File

@ -40,3 +40,4 @@ URI.
# value.
DEFAULT = 'placement.undefined_code'
INVENTORY_INUSE = 'placement.inventory.inuse'
CONCURRENT_UPDATE = 'placement.concurrent_update'

110
nova/api/openstack/placement/handlers/allocation.py
View File

@ -19,6 +19,7 @@ from oslo_utils import encodeutils
from oslo_utils import timeutils
import webob
from nova.api.openstack.placement import errors
from nova.api.openstack.placement import exception
from nova.api.openstack.placement import microversion
from nova.api.openstack.placement.objects import resource_provider as rp_obj
@ -72,7 +73,9 @@ def _serialize_allocations_for_consumer(allocations, want_version):
},
# project_id and user_id are added with microverion 1.12
'project_id': PROJECT_ID,
'user_id': USER_ID
'user_id': USER_ID,
# Generation for consumer >= 1.28
'consumer_generation': 1
}
"""
allocation_data = collections.defaultdict(dict)
@ -90,17 +93,21 @@ def _serialize_allocations_for_consumer(allocations, want_version):
if allocations and want_version.matches((1, 12)):
# We're looking at a list of allocations by consumer id so project and
# user are consistent across the list
project_id = allocations[0].consumer.project.external_id
user_id = allocations[0].consumer.user.external_id
consumer = allocations[0].consumer
project_id = consumer.project.external_id
user_id = consumer.user.external_id
result['project_id'] = project_id
result['user_id'] = user_id
show_consumer_gen = want_version.matches((1, 28))
if show_consumer_gen:
result['consumer_generation'] = consumer.generation
return result
def _serialize_allocations_for_resource_provider(allocations,
resource_provider):
resource_provider,
want_version):
"""Turn a list of allocations into a dict by consumer id.
{'resource_provider_generation': GENERATION,
@ -109,16 +116,21 @@ def _serialize_allocations_for_resource_provider(allocations,
'resources': {
'DISK_GB': 4,
'VCPU': 2
}
},
# Generation for consumer >= 1.28
'consumer_generation': 0
},
CONSUMER_ID_2: {
'resources': {
'DISK_GB': 6,
'VCPU': 3
}
},
# Generation for consumer >= 1.28
'consumer_generation': 0
}
}
"""
show_consumer_gen = want_version.matches((1, 28))
allocation_data = collections.defaultdict(dict)
for allocation in allocations:
key = allocation.consumer.uuid
@ -128,6 +140,12 @@ def _serialize_allocations_for_resource_provider(allocations,
resource_class = allocation.resource_class
allocation_data[key]['resources'][resource_class] = allocation.used
if show_consumer_gen:
consumer_gen = None
if allocation.consumer is not None:
consumer_gen = allocation.consumer.generation
allocation_data[key]['consumer_generation'] = consumer_gen
result = {'allocations': allocation_data}
result['resource_provider_generation'] = resource_provider.generation
return result
@ -187,7 +205,8 @@ def list_for_resource_provider(req):
allocs = rp_obj.AllocationList.get_all_by_resource_provider(context, rp)
output = _serialize_allocations_for_resource_provider(allocs, rp)
output = _serialize_allocations_for_resource_provider(
allocs, rp, want_version)
last_modified = _last_modified_from_allocations(allocs, want_version)
allocations_json = jsonutils.dumps(output)
@ -202,7 +221,8 @@ def list_for_resource_provider(req):
def _new_allocations(context, resource_provider_uuid, consumer_uuid,
resources, project_id, user_id):
resources, project_id, user_id, consumer_generation,
want_version):
"""Create new allocation objects for a set of resources
Returns a list of Allocation objects.
@ -214,6 +234,10 @@ def _new_allocations(context, resource_provider_uuid, consumer_uuid,
:param resources: A dict of resource classes and values.
:param project_id: The project consuming the resources.
:param user_id: The user consuming the resources.
:param consumer_generation: The generation supplied by the user when
PUT/POST'ing allocations. May be None if
the microversion is <1.28
:param want_version: The microversion object from the context.
"""
allocations = []
try:
@ -225,7 +249,8 @@ def _new_allocations(context, resource_provider_uuid, consumer_uuid,
"that does not exist.") %
{'rp_uuid': resource_provider_uuid})
consumer = util.ensure_consumer(
context, consumer_uuid, project_id, user_id)
context, consumer_uuid, project_id, user_id, consumer_generation,
want_version)
for resource_class in resources:
allocation = rp_obj.Allocation(
resource_provider=resource_provider,
@ -258,14 +283,33 @@ def _set_allocations_for_consumer(req, schema):
# If the body includes an allocation for a resource provider
# that does not exist, raise a 400.
allocation_objects = []
for resource_provider_uuid, allocation in allocation_data.items():
new_allocations = _new_allocations(context,
resource_provider_uuid,
consumer_uuid,
allocation['resources'],
data.get('project_id'),
data.get('user_id'))
allocation_objects.extend(new_allocations)
if not allocation_data:
# The allocations are empty, which means wipe them out. Internal
# to the allocation object this is signalled by a used value of 0.
# We still need to verify the consumer's generation, though, which
# we do in _ensure_consumer()
# NOTE(jaypipes): This will only occur 1.28+. The JSONSchema will
# prevent an empty allocations object from being passed when there is
# no consumer generation, so this is safe to do.
util.ensure_consumer(context, consumer_uuid, data.get('project_id'),
data.get('user_id'), data.get('consumer_generation'),
want_version)
allocations = rp_obj.AllocationList.get_all_by_consumer_id(
context, consumer_uuid)
for allocation in allocations:
allocation.used = 0
allocation_objects.append(allocation)
else:
for resource_provider_uuid, allocation in allocation_data.items():
new_allocations = _new_allocations(context,
resource_provider_uuid,
consumer_uuid,
allocation['resources'],
data.get('project_id'),
data.get('user_id'),
data.get('consumer_generation'),
want_version)
allocation_objects.extend(new_allocations)
allocations = rp_obj.AllocationList(
context, objects=allocation_objects)
@ -286,8 +330,9 @@ def _set_allocations_for_consumer(req, schema):
_('Unable to allocate inventory: %(error)s') % {'error': exc})
except exception.ConcurrentUpdateDetected as exc:
raise webob.exc.HTTPConflict(
_('Inventory changed while attempting to allocate: %(error)s') %
{'error': exc})
_('Inventory and/or allocations changed while attempting to '
'allocate: %(error)s') % {'error': exc},
comment=errors.CONCURRENT_UPDATE)
req.response.status = 204
req.response.content_type = None
@ -309,19 +354,30 @@ def set_allocations_for_consumer(req):
@wsgi_wrapper.PlacementWsgify # noqa
@microversion.version_handler('1.12')
@microversion.version_handler('1.12', '1.27')
@util.require_content('application/json')
def set_allocations_for_consumer(req):
return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_12)
@wsgi_wrapper.PlacementWsgify # noqa
@microversion.version_handler('1.28')
@util.require_content('application/json')
def set_allocations_for_consumer(req):
return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_28)
@wsgi_wrapper.PlacementWsgify
@microversion.version_handler('1.13')
@util.require_content('application/json')
def set_allocations(req):
context = req.environ['placement.context']
context.can(policies.ALLOC_MANAGE)
data = util.extract_json(req.body, schema.POST_ALLOCATIONS_V1_13)
want_version = req.environ[microversion.MICROVERSION_ENVIRON]
want_schema = schema.POST_ALLOCATIONS_V1_13
if want_version.matches((1, 28)):
want_schema = schema.POST_ALLOCATIONS_V1_28
data = util.extract_json(req.body, want_schema)
# Create a sequence of allocation objects to be used in an
# AllocationList.create_all() call, which will mean all the changes
@ -333,6 +389,7 @@ def set_allocations(req):
project_id = data[consumer_uuid]['project_id']
user_id = data[consumer_uuid]['user_id']
allocations = data[consumer_uuid]['allocations']
consumer_generation = data[consumer_uuid].get('consumer_generation')
if allocations:
for resource_provider_uuid in allocations:
resources = allocations[resource_provider_uuid]['resources']
@ -341,7 +398,9 @@ def set_allocations(req):
consumer_uuid,
resources,
project_id,
user_id)
user_id,
consumer_generation,
want_version)
allocation_objects.extend(new_allocations)
else:
# The allocations are empty, which means wipe them out.
@ -370,8 +429,9 @@ def set_allocations(req):
_('Unable to allocate inventory: %(error)s') % {'error': exc})
except exception.ConcurrentUpdateDetected as exc:
raise webob.exc.HTTPConflict(
_('Inventory changed while attempting to allocate: %(error)s') %
{'error': exc})
_('Inventory and/or allocations changed while attempting to '
'allocate: %(error)s') % {'error': exc},
comment=errors.CONCURRENT_UPDATE)
req.response.status = 204
req.response.content_type = None

1
nova/api/openstack/placement/microversion.py
View File

@ -73,6 +73,7 @@ VERSIONS = [
'1.27', # Include all resource class inventories in `provider_summaries`
# field in response of `GET /allocation_candidates` API even if
# the resource class is not in the requested resources.
'1.28', # Add support for consumer generation
]

42
nova/api/openstack/placement/objects/consumer.py
View File

@ -86,6 +86,7 @@ def _get_consumer_by_uuid(ctx, uuid):
projects.c.external_id.label("project_external_id"),
users.c.id.label("user_id"),
users.c.external_id.label("user_external_id"),
consumers.c.generation,
consumers.c.updated_at,
consumers.c.created_at
]
@ -102,6 +103,33 @@ def _get_consumer_by_uuid(ctx, uuid):
return dict(res)
@db_api.placement_context_manager.writer
def _increment_consumer_generation(ctx, consumer):
"""Increments the supplied consumer's generation value, supplying the
consumer object which contains the currently-known generation. Returns the
newly-incremented generation.
:param ctx: `nova.context.RequestContext` that contains an oslo_db Session
:param consumer: `Consumer` whose generation should be updated.
:returns: The newly-incremented generation.
:raises nova.exception.ConcurrentUpdateDetected: if another thread updated
the same consumer's view of its allocations in between the time
when this object was originally read and the call which modified
the consumer's state (e.g. replacing allocations for a consumer)
"""
consumer_gen = consumer.generation
new_generation = consumer_gen + 1
upd_stmt = CONSUMER_TBL.update().where(sa.and_(
CONSUMER_TBL.c.id == consumer.id,
CONSUMER_TBL.c.generation == consumer_gen)).values(
generation=new_generation)
res = ctx.session.execute(upd_stmt)
if res.rowcount != 1:
raise exception.ConcurrentUpdateDetected
return new_generation
@base.VersionedObjectRegistry.register_if(False)
class Consumer(base.VersionedObject, base.TimestampedObject):
@ -110,12 +138,14 @@ class Consumer(base.VersionedObject, base.TimestampedObject):
'uuid': fields.UUIDField(nullable=False),
'project': fields.ObjectField('Project', nullable=False),
'user': fields.ObjectField('User', nullable=False),
'generation': fields.IntegerField(nullable=False),
}
@staticmethod
def _from_db_object(ctx, target, source):
target.id = source['id']
target.uuid = source['uuid']
target.generation = source['generation']
target.created_at = source['created_at']
target.updated_at = source['updated_at']
@ -147,7 +177,19 @@ class Consumer(base.VersionedObject, base.TimestampedObject):
# thing here because models.Consumer doesn't have a
# project_external_id or user_external_id attribute.
self.id = db_obj.id
self.generation = db_obj.generation
except db_exc.DBDuplicateEntry:
raise exception.ConsumerExists(uuid=self.uuid)
_create_in_db(self._context)
self.obj_reset_changes()
def increment_generation(self):
"""Increments the consumer's generation.
:raises nova.exception.ConcurrentUpdateDetected: if another thread
updated the same consumer's view of its allocations in between the
time when this object was originally read and the call which
modified the consumer's state (e.g. replacing allocations for a
consumer)
"""
self.generation = _increment_consumer_generation(self._context, self)

29
nova/api/openstack/placement/objects/resource_provider.py
View File

@ -1663,6 +1663,12 @@ def _check_capacity_exceeded(ctx, allocs):
for alloc in allocs:
rc_id = _RC_CACHE.id_from_string(alloc.resource_class)
rp_uuid = alloc.resource_provider.uuid
if rp_uuid not in res_providers:
res_providers[rp_uuid] = alloc.resource_provider
amount_needed = alloc.used
# No use checking usage if we're not asking for anything
if amount_needed == 0:
continue
key = (rp_uuid, rc_id)
try:
usage = usage_map[key]
@ -1671,7 +1677,6 @@ def _check_capacity_exceeded(ctx, allocs):
raise exception.InvalidInventory(
resource_class=alloc.resource_class,
resource_provider=rp_uuid)
amount_needed = alloc.used
allocation_ratio = usage['allocation_ratio']
min_unit = usage['min_unit']
max_unit = usage['max_unit']
@ -1710,8 +1715,6 @@ def _check_capacity_exceeded(ctx, allocs):
raise exception.InvalidAllocationCapacityExceeded(
resource_class=alloc.resource_class,
resource_provider=rp_uuid)
if rp_uuid not in res_providers:
res_providers[rp_uuid] = alloc.resource_provider
return res_providers
@ -1885,6 +1888,8 @@ class AllocationList(base.ObjectListBase, base.VersionedObject):
:raises `InvalidAllocationConstraintsViolated` if any of the
`step_size`, `min_unit` or `max_unit` constraints in an
inventory will be violated by any one of the allocations.
:raises `ConcurrentUpdateDetected` if a generation for a resource
provider or consumer failed its increment check.
"""
_ensure_rc_cache(context)
# Make sure that all of the allocations are new.
@ -1919,19 +1924,17 @@ class AllocationList(base.ObjectListBase, base.VersionedObject):
# removing different allocations in the same request.
# _check_capacity_exceeded will raise a ResourceClassNotFound # if any
# allocation is using a resource class that does not exist.
visited_rps = _check_capacity_exceeded(context,
[alloc for alloc in
allocs if alloc.used > 0])
visited_consumers = {}
visited_rps = _check_capacity_exceeded(context, allocs)
for alloc in allocs:
if alloc.consumer.id not in visited_consumers:
visited_consumers[alloc.consumer.id] = alloc.consumer
# If alloc.used is set to zero that is a signal that we don't want
# to (re-)create any allocations for this resource class.
# _delete_current_allocs has already wiped out allocations so all
# that's being done here is adding the resource provider to
# visited_rps so its generation will be checked at the end of the
# transaction.
# _delete_current_allocs has already wiped out allocations so just
# continue
if alloc.used == 0:
rp = alloc.resource_provider
visited_rps[rp.uuid] = rp
continue
consumer_id = alloc.consumer.uuid
rp = alloc.resource_provider
@ -1950,6 +1953,8 @@ class AllocationList(base.ObjectListBase, base.VersionedObject):
# changes always happen atomically.
for rp in visited_rps.values():
rp.generation = _increment_provider_generation(context, rp)
for consumer in visited_consumers.values():
consumer.increment_generation()
@classmethod
def get_all_by_resource_provider(cls, context, rp):

70
nova/api/openstack/placement/rest_api_version_history.rst
View File

@ -336,3 +336,73 @@ resource provider inventory to be equal to total.
Include all resource class inventories in the ``provider_summaries`` field in
response of the ``GET /allocation_candidates`` API even if the resource class
is not in the requested resources.
1.28 Consumer generation support
--------------------------------
A new generation field has been added to the consumer concept. Consumers are
the actors that are allocated resources in the placement API. When an
allocation is created, a consumer UUID is specified, along with a project and
user ID (after microversion 1.8).
The consumer generation facilitates safe concurrent modification of an
allocation.
A consumer generation is now returned from the following URIs:
``GET /resource_providers/{uuid}/allocations``
The response continues to be a dict with a key of ``allocations``, which itself
is a dict, keyed by consumer UUID, of allocations against the resource
provider. For each of those dicts, a ``consumer_generation`` field will now be
shown.
``GET /allocations/{consumer_uuid}``
The response continues to be a dict with a key of ``allocations``, which
itself is a dict, keyed by resource provider UUID, of allocations being
consumed by the consumer with the ``{consumer_uuid}``. The top-level dict will
also now contain a ``consumer_generation`` field.
The value of the ``consumer_generation`` field will be an unsigned integer.
The ``PUT /allocations/{consumer_uuid}`` URI has been modified to now require a
``consumer_generation`` field in the request payload. This field is required to
be ``null`` if the caller expects that there are no allocations already
existing for the consumer. Otherwise, it should contain the integer generation
that the caller understands the consumer to be at the time of the call.
A ``409 Conflict`` will be returned from the ``PUT
/allocations/{consumer_uuid}`` if there was a mismatch between the supplied
generation and the consumer's generation as known by the server. Similarly, a
``409 Conflict`` will be returned if during the course of replacing the
consumer's allocations another process concurrently changed the consumer's
allocations. This allows the caller to react to the concurrent write by
re-reading the consumer's allocations and re-issuing the call to replace
allocations as needed.
The ``PUT /allocations/{consumer_uuid}`` URI has also been modified to accept
an empty allocations object, thereby bringing it to parity with the behaviour
of ``POST /allocations``, which uses an empty allocations object to indicate
that the allocations for a particular consumer should be removed. Passing an
empty allocations object along with a ``consumer_generation`` makes ``PUT
/allocations/{consumer_uuid}`` a **safe** way to delete allocations for a
consumer. The ``DELETE /allocations/{consumer_uuid}`` URI remains unsafe to
call in deployments where multiple callers may simultaneously be attempting to
modify a consumer's allocations.
The ``POST /allocations`` URI variant has also been changed to require a
``consumer_generation`` field in the request payload **for each consumer
involved in the request**. Similar responses to ``PUT
/allocations/{consumer_uuid}`` are returned when any of the consumers
generations conflict with the server's view of those consumers or if any of the
consumers involved in the request are modified by another process.
**WARNING**:
In all cases, it is absolutely **NOT SAFE** to create and modify allocations
for a consumer using different microversions where one of the microversions is
prior to 1.28. The only way to safely modify allocations for a consumer and
satisfy expectations you have regarding the prior existence (or lack of
existence) of those allocations is to always use microversion 1.28+ when
calling allocations API endpoints.

25
nova/api/openstack/placement/schemas/allocation.py
View File

@ -139,3 +139,28 @@ POST_ALLOCATIONS_V1_13 = {
"^[0-9a-fA-F-]{36}$": DELETABLE_ALLOCATIONS
}
}
# A required consumer generation was added to the top-level dict in this
# version of PUT /allocations/{consumer_uuid}. In addition, the PUT
# /allocations/{consumer_uuid}/now allows for empty allocations (indicating the
# allocations are being removed)
ALLOCATION_SCHEMA_V1_28 = copy.deepcopy(DELETABLE_ALLOCATIONS)
ALLOCATION_SCHEMA_V1_28['properties']['consumer_generation'] = {
"type": ["integer", "null"],
"additionalProperties": False
}
ALLOCATION_SCHEMA_V1_28['required'].append("consumer_generation")
# A required consumer generation was added to the allocations dicts in this
# version of POST /allocations
REQUIRED_GENERATION_ALLOCS_POST = copy.deepcopy(DELETABLE_ALLOCATIONS)
alloc_props = REQUIRED_GENERATION_ALLOCS_POST['properties']
alloc_props['consumer_generation'] = {
"type": ["integer", "null"],
"additionalProperties": False
}
REQUIRED_GENERATION_ALLOCS_POST['required'].append("consumer_generation")
POST_ALLOCATIONS_V1_28 = copy.deepcopy(POST_ALLOCATIONS_V1_13)
POST_ALLOCATIONS_V1_28["patternProperties"] = {
"^[0-9a-fA-F-]{36}$": REQUIRED_GENERATION_ALLOCS_POST
}

28
nova/api/openstack/placement/util.py
View File

@ -577,7 +577,8 @@ def parse_qs_request_groups(req):
return by_suffix
def ensure_consumer(ctx, consumer_uuid, project_id, user_id):
def ensure_consumer(ctx, consumer_uuid, project_id, user_id,
consumer_generation, want_version):
"""Ensures there are records in the consumers, projects and users table for
the supplied external identifiers.
@ -587,7 +588,13 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id):
:param consumer_uuid: The uuid of the consumer of the resources.
:param project_id: The external ID of the project consuming the resources.
:param user_id: The external ID of the user consuming the resources.
:param consumer_generation: The generation provided by the user for this
consumer.
:param want_version: the microversion matcher.
:raises webob.exc.HTTPConflict if consumer generation is required and there
was a mismatch
"""
requires_consumer_generation = want_version.matches((1, 28))
if project_id is None:
project_id = CONF.placement.incomplete_consumer_project_id
user_id = CONF.placement.incomplete_consumer_user_id
@ -614,7 +621,26 @@ def ensure_consumer(ctx, consumer_uuid, project_id, user_id):
try:
consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid)
if requires_consumer_generation:
if consumer.generation != consumer_generation:
raise webob.exc.HTTPConflict(
_('consumer generation conflict - '
'expected %(expected_gen)s but got %(got_gen)s') %
{
'expected_gen': consumer.generation,
'got_gen': consumer_generation,
})
except exception.NotFound:
# If we are attempting to modify or create allocations after 1.26, we
# need a consumer generation specified. The user must have specified
# None for the consumer generation if we get here, since there was no
# existing consumer with this UUID and therefore the user should be
# indicating that they expect the consumer did not exist.
if requires_consumer_generation:
if consumer_generation is not None:
raise webob.exc.HTTPConflict(
_('consumer generation conflict - '
'expected None but got %s') % consumer_generation)
# No such consumer. This is common for new allocations. Create the
# consumer record
try:

168
nova/tests/functional/api/openstack/placement/gabbits/allocations-1.28.yaml Normal file
View File

@ -0,0 +1,168 @@
fixtures:
- AllocationFixture
defaults:
request_headers:
x-auth-token: admin
accept: application/json
content-type: application/json
openstack-api-version: placement 1.28
#
# Scenarios to test
# Start with no consumers
# old, no CG = success, consumer gets created
# new, no CG = fail, due to schema
# new, CG=None = success, consumer gets created
# new, CG=<any> = fail
# Create an allocation, and with it, a consumer
# Now create another allocation
# old, no CG = success
# new, CG=None = fail
# new, CG !match = fail
# new, get CG from /allocations
# new, CG matches = success
tests:
- name: old version no gen no existing
PUT: /allocations/11111111-1111-1111-1111-111111111111
request_headers:
openstack-api-version: placement 1.27
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
status: 204
- name: new version no gen no existing
PUT: /allocations/22222222-2222-2222-2222-222222222222
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
status: 400
response_strings:
- JSON does not validate
- name: new version gen is None no existing
PUT: /allocations/22222222-2222-2222-2222-222222222222
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: null
status: 204
- name: new version any gen no existing
PUT: /allocations/33333333-3333-3333-3333-333333333333
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: 33
status: 409
response_strings:
- consumer generation conflict
# Now create an allocation for a specific consumer
- name: put an allocation
PUT: /allocations/44444444-4444-4444-4444-444444444444
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: null
status: 204
- name: new version no gen existing
PUT: /allocations/44444444-4444-4444-4444-444444444444
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: null
status: 409
response_strings:
- consumer generation conflict
- name: get the current consumer generation
GET: /allocations/44444444-4444-4444-4444-444444444444
status: 200
- name: new version matching gen existing
PUT: /allocations/44444444-4444-4444-4444-444444444444
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: $HISTORY["get the current consumer generation"].$RESPONSE["consumer_generation"]
status: 204
- name: new version mismatch gen existing
PUT: /allocations/44444444-4444-4444-4444-444444444444
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: 12
status: 409
response_strings:
- consumer generation conflict
- name: old version no gen existing
PUT: /allocations/44444444-4444-4444-4444-444444444444
request_headers:
openstack-api-version: placement 1.27
data:
allocations:
$ENVIRON['RP_UUID']:
resources:
DISK_GB: 10
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
status: 204
- name: new version serialization contains consumer generation
GET: /allocations/44444444-4444-4444-4444-444444444444
status: 200
response_json_paths:
$.consumer_generation: /^\d+$/
- name: empty allocations dict now possible in PUT /allocations/{consumer_uuid}
PUT: /allocations/44444444-4444-4444-4444-444444444444
data:
allocations: {}
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
consumer_generation: $HISTORY["new version serialization contains consumer generation"].$RESPONSE["consumer_generation"]
status: 204
- name: should now return no allocations for this consumer
GET: /allocations/44444444-4444-4444-4444-444444444444
status: 200
response_json_paths:
$.allocations.`len`: 0

4
nova/tests/functional/api/openstack/placement/gabbits/allocations-policy.yaml
View File

@ -38,7 +38,7 @@ tests:
total: 96
status: 200
- name: update allocation for consumer
- name: create allocation for consumer
PUT: /allocations/a0b15655-273a-4b3d-9792-2e579b7d5ad9
data:
allocations:
@ -46,6 +46,7 @@ tests:
resources:
VCPU: 1
DISK_GB: 20
consumer_generation: null
project_id: 42a32c07-3eeb-4401-9373-68a8cdca6784
user_id: 66cb2f29-c86d-47c3-8af5-69ae7b778c70
status: 204
@ -60,6 +61,7 @@ tests:
POST: /allocations
data:
a0b15655-273a-4b3d-9792-2e579b7d5ad9:
consumer_generation: 1
project_id: 42a32c07-3eeb-4401-9373-68a8cdca6784
user_id: 66cb2f29-c86d-47c3-8af5-69ae7b778c70
allocations:

86
nova/tests/functional/api/openstack/placement/gabbits/allocations-post.yaml
View File

@ -286,3 +286,89 @@ tests:
status: 400
response_strings:
- No such resource class CUSTOM_PONY
- name: fail missing consumer generation >= 1.28
POST: /allocations
request_headers:
openstack-api-version: placement 1.28
data:
$ENVIRON['INSTANCE_UUID']:
allocations:
$HISTORY['rp compute02'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 1024
VCPU: 2
$HISTORY['rp storage01'].$RESPONSE['uuid']:
resources:
DISK_GB: 5
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
$ENVIRON['CONSUMER_UUID']:
allocations:
$HISTORY['rp compute01'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 2049
VCPU: 2
project_id: $ENVIRON['PROJECT_ID_ALT']
user_id: $ENVIRON['USER_ID_ALT']
status: 400
response_strings:
- JSON does not validate
- name: fail incorrect consumer generation >= 1.28
POST: /allocations
request_headers:
openstack-api-version: placement 1.28
data:
$ENVIRON['INSTANCE_UUID']:
allocations:
$HISTORY['rp compute02'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 1024
VCPU: 1
$HISTORY['rp storage01'].$RESPONSE['uuid']:
resources:
DISK_GB: 4
consumer_generation: 1
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
$ENVIRON['CONSUMER_UUID']:
allocations:
$HISTORY['rp compute01'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 1024
VCPU: 1
consumer_generation: 1
project_id: $ENVIRON['PROJECT_ID_ALT']
user_id: $ENVIRON['USER_ID_ALT']
status: 409
response_strings:
- consumer generation conflict - expected 3 but got 1
- name: change allocations for existing providers >= 1.28
POST: /allocations
request_headers:
openstack-api-version: placement 1.28
data:
$ENVIRON['INSTANCE_UUID']:
allocations:
$HISTORY['rp compute02'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 1024
VCPU: 1
$HISTORY['rp storage01'].$RESPONSE['uuid']:
resources:
DISK_GB: 4
consumer_generation: 3
project_id: $ENVIRON['PROJECT_ID']
user_id: $ENVIRON['USER_ID']
$ENVIRON['CONSUMER_UUID']:
allocations:
$HISTORY['rp compute01'].$RESPONSE['uuid']:
resources:
MEMORY_MB: 1024
VCPU: 1
consumer_generation: 1
project_id: $ENVIRON['PROJECT_ID_ALT']
user_id: $ENVIRON['USER_ID_ALT']
status: 204

4
nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml
View File

@ -41,13 +41,13 @@ tests:
response_json_paths:
$.errors[0].title: Not Acceptable
- name: latest microversion is 1.27
- name: latest microversion is 1.28
GET: /
request_headers:
openstack-api-version: placement latest
response_headers:
vary: /openstack-api-version/
openstack-api-version: placement 1.27
openstack-api-version: placement 1.28
- name: other accept header bad version
GET: /

116
nova/tests/unit/api/openstack/placement/test_util.py
View File

@ -29,6 +29,7 @@ import six
from nova.api.openstack.placement import exception
from nova.api.openstack.placement import lib as pl
from nova.api.openstack.placement import microversion
from nova.api.openstack.placement.objects import consumer as consumer_obj
from nova.api.openstack.placement.objects import project as project_obj
from nova.api.openstack.placement.objects import resource_provider as rp_obj
from nova.api.openstack.placement.objects import user as user_obj
@ -927,14 +928,31 @@ class TestEnsureConsumer(testtools.TestCase):
self.consumer_id = uuidsentinel.consumer
self.project_id = uuidsentinel.project
self.user_id = uuidsentinel.user
mv_parsed = microversion_parse.Version(1, 27)
mv_parsed.max_version = microversion_parse.parse_version_string(
microversion.max_version_string())
mv_parsed.min_version = microversion_parse.parse_version_string(
microversion.min_version_string())
self.before_version = mv_parsed
mv_parsed = microversion_parse.Version(1, 28)
mv_parsed.max_version = microversion_parse.parse_version_string(
microversion.max_version_string())
mv_parsed.min_version = microversion_parse.parse_version_string(
microversion.min_version_string())
self.after_version = mv_parsed
def test_no_existing_project_user_consumer(self):
def test_no_existing_project_user_consumer_before_gen_success(self):
"""Tests that we don't require a consumer_generation=None before the
appropriate microversion.
"""
self.mock_project_get.side_effect = exception.NotFound
self.mock_user_get.side_effect = exception.NotFound
self.mock_consumer_get.side_effect = exception.NotFound
consumer_gen = 1 # should be ignored
util.ensure_consumer(
self.ctx, self.consumer_id, self.project_id, self.user_id)
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.before_version)
self.mock_project_get.assert_called_once_with(
self.ctx, self.project_id)
@ -946,6 +964,44 @@ class TestEnsureConsumer(testtools.TestCase):
self.mock_user_create.assert_called_once()
self.mock_consumer_create.assert_called_once()
def test_no_existing_project_user_consumer_after_gen_success(self):
"""Tests that we require a consumer_generation=None after the
appropriate microversion.
"""
self.mock_project_get.side_effect = exception.NotFound
self.mock_user_get.side_effect = exception.NotFound
self.mock_consumer_get.side_effect = exception.NotFound
consumer_gen = None # should NOT be ignored (and None is expected)
util.ensure_consumer(
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.after_version)
self.mock_project_get.assert_called_once_with(
self.ctx, self.project_id)
self.mock_user_get.assert_called_once_with(
self.ctx, self.user_id)
self.mock_consumer_get.assert_called_once_with(
self.ctx, self.consumer_id)
self.mock_project_create.assert_called_once()
self.mock_user_create.assert_called_once()
self.mock_consumer_create.assert_called_once()
def test_no_existing_project_user_consumer_after_gen_fail(self):
"""Tests that we require a consumer_generation=None after the
appropriate microversion and that None is the expected value.
"""
self.mock_project_get.side_effect = exception.NotFound
self.mock_user_get.side_effect = exception.NotFound
self.mock_consumer_get.side_effect = exception.NotFound
consumer_gen = 1 # should NOT be ignored (and 1 is not expected)
self.assertRaises(
webob.exc.HTTPConflict,
util.ensure_consumer,
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.after_version)
def test_no_existing_project_user_consumer_use_incomplete(self):
"""Verify that if the project_id arg is None, that we fall back to the
CONF options for incomplete project and user ID.
@ -954,8 +1010,10 @@ class TestEnsureConsumer(testtools.TestCase):
self.mock_user_get.side_effect = exception.NotFound
self.mock_consumer_get.side_effect = exception.NotFound
consumer_gen = None # should NOT be ignored (and None is expected)
util.ensure_consumer(
self.ctx, self.consumer_id, None, None)
self.ctx, self.consumer_id, None, None,
consumer_gen, self.before_version)
self.mock_project_get.assert_called_once_with(
self.ctx, CONF.placement.incomplete_consumer_project_id)
@ -967,9 +1025,10 @@ class TestEnsureConsumer(testtools.TestCase):
self.mock_user_create.assert_called_once()
self.mock_consumer_create.assert_called_once()
def test_existing_project_user_no_existing_consumer(self):
def test_existing_project_no_existing_consumer_before_gen_success(self):
"""Check that if we find an existing project and user, that we use
those found objects in creating the consumer.
those found objects in creating the consumer. Do not require a consumer
generation before the appropriate microversion.
"""
proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id)
self.mock_project_get.return_value = proj
@ -977,9 +1036,54 @@ class TestEnsureConsumer(testtools.TestCase):
self.mock_user_get.return_value = user
self.mock_consumer_get.side_effect = exception.NotFound
consumer_gen = None # should be ignored
util.ensure_consumer(
self.ctx, self.consumer_id, self.project_id, self.user_id)
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.before_version)
self.mock_project_create.assert_not_called()
self.mock_user_create.assert_not_called()
self.mock_consumer_create.assert_called_once()
def test_existing_consumer_after_gen_matches_supplied_gen(self):
"""Tests that we require a consumer_generation after the
appropriate microversion and that when the consumer already exists,
then we ensure a matching generation is supplied
"""
proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id)
self.mock_project_get.return_value = proj
user = user_obj.User(self.ctx, id=1, external_id=self.user_id)
self.mock_user_get.return_value = user
consumer = consumer_obj.Consumer(
self.ctx, id=1, project=proj, user=user, generation=2)
self.mock_consumer_get.return_value = consumer
consumer_gen = 2 # should NOT be ignored (and 2 is expected)
util.ensure_consumer(
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.after_version)
self.mock_project_create.assert_not_called()
self.mock_user_create.assert_not_called()
self.mock_consumer_create.assert_not_called()
def test_existing_consumer_after_gen_fail(self):
"""Tests that we require a consumer_generation after the
appropriate microversion and that when the consumer already exists,
then we raise a 400 when there is a mismatch on the existing
generation.
"""
proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id)
self.mock_project_get.return_value = proj
user = user_obj.User(self.ctx, id=1, external_id=self.user_id)
self.mock_user_get.return_value = user
consumer = consumer_obj.Consumer(
self.ctx, id=1, project=proj, user=user, generation=42)
self.mock_consumer_get.return_value = consumer
consumer_gen = 2 # should NOT be ignored (and 2 is NOT expected)
self.assertRaises(
webob.exc.HTTPConflict,
util.ensure_consumer,
self.ctx, self.consumer_id, self.project_id, self.user_id,
consumer_gen, self.after_version)

30
placement-api-ref/source/allocations.inc
View File

@ -37,12 +37,20 @@ Request
.. rest_parameters:: parameters.yaml
- consumer_uuid: consumer_uuid_body
- consumer_generation: consumer_generation
- project_id: project_id_body
- user_id: user_id_body
- allocations: allocations_dict_empty
- resources: resources
Request Example
Request example (microversions 1.28 - )
---------------------------------------
.. literalinclude:: ./samples/allocations/manage-allocations-request-1.28.json
:language: javascript
Request example (microversions 1.13 - 1.27)
-------------------------------------------
.. literalinclude:: ./samples/allocations/manage-allocations-request.json
:language: javascript
@ -81,11 +89,18 @@ Response
- allocations: allocations_by_resource_provider
- generation: resource_provider_generation
- resources: resources
- consumer_generation: consumer_generation
- project_id: project_id_body_1_12
- user_id: user_id_body_1_12
Response Example
----------------
Response Example (1.28 - )
--------------------------
.. literalinclude:: ./samples/allocations/get-allocations-1.28.json
:language: javascript
Response Example (1.12 - 1.27)
------------------------------
.. literalinclude:: ./samples/allocations/get-allocations.json
:language: javascript
@ -116,13 +131,20 @@ Request (microversions 1.12 - )
- consumer_uuid: consumer_uuid
- allocations: allocations_dict
- resources: resources
- consumer_generation: consumer_generation
- project_id: project_id_body
- user_id: user_id_body
- generation: resource_provider_generation_optional
Request example (microversions 1.12 - )
Request example (microversions 1.28 - )
---------------------------------------
.. literalinclude:: ./samples/allocations/update-allocations-request-1.28.json
:language: javascript
Request example (microversions 1.12 - 1.27)
-------------------------------------------
.. literalinclude:: ./samples/allocations/update-allocations-request-1.12.json
:language: javascript

8
placement-api-ref/source/parameters.yaml
View File

@ -314,6 +314,14 @@ capacity:
required: true
description: >
The amount of the resource that the provider can accommodate.
consumer_generation:
type: integer
in: body
required: true
min_version: 1.28
description: >
The generation of the consumer. Should be set to ``None`` when indicating
the the caller expects the consumer does not yet exist.
consumer_uuid_body:
<<: *consumer_uuid
in: body

20
placement-api-ref/source/samples/allocations/get-allocations-1.28.json Normal file
View File

@ -0,0 +1,20 @@
{
"allocations": {
"92637880-2d79-43c6-afab-d860886c6391": {
"generation": 2,
"resources": {
"DISK_GB": 5
}
},
"ba8e1ef8-7fa3-41a4-9bb4-d7cb2019899b": {
"generation": 8,
"resources": {
"MEMORY_MB": 512,
"VCPU": 2
}
}
},
"consumer_generation": 1,
"project_id": "7e67cbf7-7c38-4a32-b85b-0739c690991a",
"user_id": "067f691e-725a-451a-83e2-5c3d13e1dffc"
}

35
placement-api-ref/source/samples/allocations/manage-allocations-request-1.28.json Normal file
View File

@ -0,0 +1,35 @@
{
"30328d13-e299-4a93-a102-61e4ccabe474": {
"consumer_generation": 1,
"project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"allocations": {
"e10927c4-8bc9-465d-ac60-d2f79f7e4a00": {
"resources": {
"VCPU": 2,
"MEMORY_MB": 3
}
}
}
},
"71921e4e-1629-4c5b-bf8d-338d915d2ef3": {
"consumer_generation": 1,
"project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"allocations": {}
},
"48c1d40f-45d8-4947-8d46-52b4e1326df8": {
"consumer_generation": 1,
"project_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"user_id": "131d4efb-abc0-4872-9b92-8c8b9dc4320f",
"allocations": {
"e10927c4-8bc9-465d-ac60-d2f79f7e4a00": {
"resources": {
"VCPU": 4,
"MEMORY_MB": 5
}
}
}
}
}

18
placement-api-ref/source/samples/allocations/update-allocations-request-1.28.json Normal file
View File

@ -0,0 +1,18 @@
{
"allocations": {
"4e061c03-611e-4caa-bf26-999dcff4284e": {
"resources": {
"DISK_GB": 20
}
},
"89873422-1373-46e5-b467-f0c5e6acf08f": {
"resources": {
"MEMORY_MB": 1024,
"VCPU": 1
}
}
},
"consumer_generation": 1,
"user_id": "66cb2f29-c86d-47c3-8af5-69ae7b778c70",
"project_id": "42a32c07-3eeb-4401-9373-68a8cdca6784"
}

17
releasenotes/notes/consumer_generation-f576ac2594b24e2e.yaml Normal file
View File

@ -0,0 +1,17 @@
---
features:
- |
Adds a new ``generation`` column to the consumers table. This value is
incremented every time allocations are made for a consumer. The new
placement microversion 1.28 requires that all ``POST`` and ``PUT
/allocations`` requests now include the ``consumer_generation`` parameter
to ensure that if two processes are allocating resources for the same
consumer, the second one to post doesn't overwrite the first. If there is a
mismatch between the ``consumer_generation`` in the request and the current
value in the database, the allocation will fail, and a 409 Conflict
response will be returned. The calling process must then get the
allocations for that consumer by calling ``GET /allocations/{consumer}``.
That response will now contain, in addition to the allocations, the current
value of the generation value for that consumer. The calling process should
then combine the existing allocations with the ones it is trying to post,
and re-submit with the current consumer_generation.