From 70a2879b2c75377f728f8faec8bd581613061230 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Thu, 31 Jan 2019 10:50:10 +0000 Subject: [PATCH] Delete the placement code This finalizes the removal of the placement code from nova. This change primarily removes code and makes fixes to cmd, test and migration tooling to adapt to the removal. Placement tests and documention were already removed in early patches. A database migration that calls consumer_obj.create_incomplete_consumers in nova-manage has been removed. A functional test which confirms the default incomplete consumer user and project id has been changes so its its use of conf.placement.incomplete_* (now removed) is replaced with a constant. The placement server, running in the functional test, provides its own config. placement-related configuration is updated to only register those opts which are relevant on the nova side. This mostly means ksa-related opts. placement-database configuration is removed from nova/conf/database. tox.ini is updated to remove the group_regex required by the placement gabbi tests. This should probably have gone when the placement functional tests went, but was overlooked. A release note is added which describes that this is cleanup, the main action already happened, but points people to the nova to placement upgrade instructions in case they haven't done it yet. Change-Id: I4181f39dea7eb10b84e6f5057938767b3e422aff --- .gitignore | 1 - doc/source/conf.py | 1 - nova/api/openstack/placement/__init__.py | 0 nova/api/openstack/placement/auth.py | 102 - nova/api/openstack/placement/context.py | 52 - nova/api/openstack/placement/db_api.py | 48 - nova/api/openstack/placement/deploy.py | 120 - nova/api/openstack/placement/direct.py | 94 - nova/api/openstack/placement/errors.py | 48 - nova/api/openstack/placement/exception.py | 207 - nova/api/openstack/placement/fault_wrap.py | 48 - nova/api/openstack/placement/handler.py | 231 - .../openstack/placement/handlers/__init__.py | 0 .../openstack/placement/handlers/aggregate.py | 133 - .../placement/handlers/allocation.py | 576 --- .../handlers/allocation_candidate.py | 332 -- .../openstack/placement/handlers/inventory.py | 467 -- .../openstack/placement/handlers/reshaper.py | 129 - .../placement/handlers/resource_class.py | 241 - .../placement/handlers/resource_provider.py | 308 -- nova/api/openstack/placement/handlers/root.py | 54 - .../api/openstack/placement/handlers/trait.py | 270 -- .../api/openstack/placement/handlers/usage.py | 120 - nova/api/openstack/placement/lib.py | 53 - nova/api/openstack/placement/microversion.py | 172 - .../openstack/placement/objects/__init__.py | 0 .../openstack/placement/objects/consumer.py | 257 - .../openstack/placement/objects/project.py | 92 - .../placement/objects/resource_provider.py | 4282 ----------------- nova/api/openstack/placement/objects/user.py | 92 - .../openstack/placement/policies/__init__.py | 39 - .../openstack/placement/policies/aggregate.py | 53 - .../placement/policies/allocation.py | 92 - .../policies/allocation_candidate.py | 38 - nova/api/openstack/placement/policies/base.py | 42 - .../openstack/placement/policies/inventory.py | 95 - .../openstack/placement/policies/reshaper.py | 38 - .../placement/policies/resource_class.py | 86 - .../placement/policies/resource_provider.py | 86 - .../api/openstack/placement/policies/trait.py | 120 - .../api/openstack/placement/policies/usage.py | 54 - nova/api/openstack/placement/policy.py | 94 - nova/api/openstack/placement/requestlog.py | 87 - .../placement/resource_class_cache.py | 154 - .../placement/rest_api_version_history.rst | 518 -- .../openstack/placement/schemas/__init__.py | 0 .../openstack/placement/schemas/aggregate.py | 42 - .../openstack/placement/schemas/allocation.py | 169 - .../placement/schemas/allocation_candidate.py | 78 - .../api/openstack/placement/schemas/common.py | 22 - .../openstack/placement/schemas/inventory.py | 93 - .../openstack/placement/schemas/reshaper.py | 47 - .../placement/schemas/resource_class.py | 33 - .../placement/schemas/resource_provider.py | 106 - nova/api/openstack/placement/schemas/trait.py | 56 - nova/api/openstack/placement/schemas/usage.py | 33 - nova/api/openstack/placement/util.py | 697 --- nova/api/openstack/placement/wsgi.py | 120 - nova/api/openstack/placement/wsgi_wrapper.py | 38 - nova/cmd/manage.py | 22 +- nova/conf/database.py | 54 - nova/conf/placement.py | 60 +- nova/config.py | 2 - nova/db/sqlalchemy/migration.py | 9 +- nova/hacking/checks.py | 9 +- nova/rc_fields.py | 70 - nova/tests/functional/test_nova_manage.py | 7 +- nova/tests/unit/policy_fixture.py | 30 - nova/tests/unit/test_conf.py | 5 - nova/tests/unit/test_nova_manage.py | 12 +- .../placement-deleted-a79ad405f428a5f8.yaml | 13 + setup.cfg | 3 - tox.ini | 7 +- 73 files changed, 30 insertions(+), 11933 deletions(-) delete mode 100644 nova/api/openstack/placement/__init__.py delete mode 100644 nova/api/openstack/placement/auth.py delete mode 100644 nova/api/openstack/placement/context.py delete mode 100644 nova/api/openstack/placement/db_api.py delete mode 100644 nova/api/openstack/placement/deploy.py delete mode 100644 nova/api/openstack/placement/direct.py delete mode 100644 nova/api/openstack/placement/errors.py delete mode 100644 nova/api/openstack/placement/exception.py delete mode 100644 nova/api/openstack/placement/fault_wrap.py delete mode 100644 nova/api/openstack/placement/handler.py delete mode 100644 nova/api/openstack/placement/handlers/__init__.py delete mode 100644 nova/api/openstack/placement/handlers/aggregate.py delete mode 100644 nova/api/openstack/placement/handlers/allocation.py delete mode 100644 nova/api/openstack/placement/handlers/allocation_candidate.py delete mode 100644 nova/api/openstack/placement/handlers/inventory.py delete mode 100644 nova/api/openstack/placement/handlers/reshaper.py delete mode 100644 nova/api/openstack/placement/handlers/resource_class.py delete mode 100644 nova/api/openstack/placement/handlers/resource_provider.py delete mode 100644 nova/api/openstack/placement/handlers/root.py delete mode 100644 nova/api/openstack/placement/handlers/trait.py delete mode 100644 nova/api/openstack/placement/handlers/usage.py delete mode 100644 nova/api/openstack/placement/lib.py delete mode 100644 nova/api/openstack/placement/microversion.py delete mode 100644 nova/api/openstack/placement/objects/__init__.py delete mode 100644 nova/api/openstack/placement/objects/consumer.py delete mode 100644 nova/api/openstack/placement/objects/project.py delete mode 100644 nova/api/openstack/placement/objects/resource_provider.py delete mode 100644 nova/api/openstack/placement/objects/user.py delete mode 100644 nova/api/openstack/placement/policies/__init__.py delete mode 100644 nova/api/openstack/placement/policies/aggregate.py delete mode 100644 nova/api/openstack/placement/policies/allocation.py delete mode 100644 nova/api/openstack/placement/policies/allocation_candidate.py delete mode 100644 nova/api/openstack/placement/policies/base.py delete mode 100644 nova/api/openstack/placement/policies/inventory.py delete mode 100644 nova/api/openstack/placement/policies/reshaper.py delete mode 100644 nova/api/openstack/placement/policies/resource_class.py delete mode 100644 nova/api/openstack/placement/policies/resource_provider.py delete mode 100644 nova/api/openstack/placement/policies/trait.py delete mode 100644 nova/api/openstack/placement/policies/usage.py delete mode 100644 nova/api/openstack/placement/policy.py delete mode 100644 nova/api/openstack/placement/requestlog.py delete mode 100644 nova/api/openstack/placement/resource_class_cache.py delete mode 100644 nova/api/openstack/placement/rest_api_version_history.rst delete mode 100644 nova/api/openstack/placement/schemas/__init__.py delete mode 100644 nova/api/openstack/placement/schemas/aggregate.py delete mode 100644 nova/api/openstack/placement/schemas/allocation.py delete mode 100644 nova/api/openstack/placement/schemas/allocation_candidate.py delete mode 100644 nova/api/openstack/placement/schemas/common.py delete mode 100644 nova/api/openstack/placement/schemas/inventory.py delete mode 100644 nova/api/openstack/placement/schemas/reshaper.py delete mode 100644 nova/api/openstack/placement/schemas/resource_class.py delete mode 100644 nova/api/openstack/placement/schemas/resource_provider.py delete mode 100644 nova/api/openstack/placement/schemas/trait.py delete mode 100644 nova/api/openstack/placement/schemas/usage.py delete mode 100644 nova/api/openstack/placement/util.py delete mode 100644 nova/api/openstack/placement/wsgi.py delete mode 100644 nova/api/openstack/placement/wsgi_wrapper.py delete mode 100644 nova/rc_fields.py create mode 100644 releasenotes/notes/placement-deleted-a79ad405f428a5f8.yaml diff --git a/.gitignore b/.gitignore index 1a3838669549..3c64ffabe0be 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,6 @@ nova/vcsversion.py tools/conf/nova.conf* doc/source/_static/nova.conf.sample doc/source/_static/nova.policy.yaml.sample -doc/source/_static/placement.policy.yaml.sample # Files created by releasenotes build releasenotes/build diff --git a/doc/source/conf.py b/doc/source/conf.py index 1220e0f5cbf1..c5a97436da6c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -55,7 +55,6 @@ sample_config_basename = '_static/nova' policy_generator_config_file = [ ('../../etc/nova/nova-policy-generator.conf', '_static/nova'), - ('../../etc/nova/placement-policy-generator.conf', '_static/placement') ] actdiag_html_image_format = 'SVG' diff --git a/nova/api/openstack/placement/__init__.py b/nova/api/openstack/placement/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/api/openstack/placement/auth.py b/nova/api/openstack/placement/auth.py deleted file mode 100644 index ff2551e26faa..000000000000 --- a/nova/api/openstack/placement/auth.py +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from keystonemiddleware import auth_token -from oslo_log import log as logging -from oslo_middleware import request_id -import webob.dec -import webob.exc - -from nova.api.openstack.placement import context - -LOG = logging.getLogger(__name__) - - -class Middleware(object): - - def __init__(self, application, **kwargs): - self.application = application - - -# NOTE(cdent): Only to be used in tests where auth is being faked. -class NoAuthMiddleware(Middleware): - """Require a token if one isn't present.""" - - def __init__(self, application): - self.application = application - - @webob.dec.wsgify - def __call__(self, req): - if req.environ['PATH_INFO'] == '/': - return self.application - - if 'X-Auth-Token' not in req.headers: - return webob.exc.HTTPUnauthorized() - - token = req.headers['X-Auth-Token'] - user_id, _sep, project_id = token.partition(':') - project_id = project_id or user_id - if user_id == 'admin': - roles = ['admin'] - else: - roles = [] - req.headers['X_USER_ID'] = user_id - req.headers['X_TENANT_ID'] = project_id - req.headers['X_ROLES'] = ','.join(roles) - return self.application - - -class PlacementKeystoneContext(Middleware): - """Make a request context from keystone headers.""" - - @webob.dec.wsgify - def __call__(self, req): - req_id = req.environ.get(request_id.ENV_REQUEST_ID) - - ctx = context.RequestContext.from_environ( - req.environ, request_id=req_id) - - if ctx.user_id is None and req.environ['PATH_INFO'] != '/': - LOG.debug("Neither X_USER_ID nor X_USER found in request") - return webob.exc.HTTPUnauthorized() - - req.environ['placement.context'] = ctx - return self.application - - -class PlacementAuthProtocol(auth_token.AuthProtocol): - """A wrapper on Keystone auth_token middleware. - - Does not perform verification of authentication tokens - for root in the API. - - """ - def __init__(self, app, conf): - self._placement_app = app - super(PlacementAuthProtocol, self).__init__(app, conf) - - def __call__(self, environ, start_response): - if environ['PATH_INFO'] == '/': - return self._placement_app(environ, start_response) - - return super(PlacementAuthProtocol, self).__call__( - environ, start_response) - - -def filter_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - - def auth_filter(app): - return PlacementAuthProtocol(app, conf) - return auth_filter diff --git a/nova/api/openstack/placement/context.py b/nova/api/openstack/placement/context.py deleted file mode 100644 index ee0786f494cd..000000000000 --- a/nova/api/openstack/placement/context.py +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_context import context -from oslo_db.sqlalchemy import enginefacade - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import policy - - -@enginefacade.transaction_context_provider -class RequestContext(context.RequestContext): - - def can(self, action, target=None, fatal=True): - """Verifies that the given action is valid on the target in this - context. - - :param action: string representing the action to be checked. - :param target: As much information about the object being operated on - as possible. The target argument should be a dict instance or an - instance of a class that fully supports the Mapping abstract base - class and deep copying. For object creation this should be a - dictionary representing the location of the object e.g. - ``{'project_id': context.project_id}``. If None, then this default - target will be considered:: - - {'project_id': self.project_id, 'user_id': self.user_id} - :param fatal: if False, will return False when an - exception.PolicyNotAuthorized occurs. - :raises nova.api.openstack.placement.exception.PolicyNotAuthorized: - if verification fails and fatal is True. - :return: returns a non-False value (not necessarily "True") if - authorized and False if not authorized and fatal is False. - """ - if target is None: - target = {'project_id': self.project_id, - 'user_id': self.user_id} - try: - return policy.authorize(self, action, target) - except exception.PolicyNotAuthorized: - if fatal: - raise - return False diff --git a/nova/api/openstack/placement/db_api.py b/nova/api/openstack/placement/db_api.py deleted file mode 100644 index 31426cbd6d52..000000000000 --- a/nova/api/openstack/placement/db_api.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Database context manager for placement database connection, kept in its -own file so the nova db_api (which has cascading imports) is not imported. -""" - -from oslo_db.sqlalchemy import enginefacade -from oslo_log import log as logging - -from nova.utils import run_once - -placement_context_manager = enginefacade.transaction_context() -LOG = logging.getLogger(__name__) - - -def _get_db_conf(conf_group): - return dict(conf_group.items()) - - -@run_once("TransactionFactory already started, not reconfiguring.", - LOG.warning) -def configure(conf): - # If [placement_database]/connection is not set in conf, then placement - # data will be stored in the nova_api database. - if conf.placement_database.connection is None: - placement_context_manager.configure( - **_get_db_conf(conf.api_database)) - else: - placement_context_manager.configure( - **_get_db_conf(conf.placement_database)) - - -def get_placement_engine(): - return placement_context_manager.writer.get_engine() - - -@enginefacade.transaction_context_provider -class DbContext(object): - """Stub class for db session handling outside of web requests.""" diff --git a/nova/api/openstack/placement/deploy.py b/nova/api/openstack/placement/deploy.py deleted file mode 100644 index 76de333ebb96..000000000000 --- a/nova/api/openstack/placement/deploy.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Deployment handling for Placmenent API.""" - -from microversion_parse import middleware as mp_middleware -import oslo_middleware -from oslo_middleware import cors - -from nova.api.openstack.placement import auth -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import fault_wrap -from nova.api.openstack.placement import handler -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider -from nova.api.openstack.placement import requestlog -from nova.api.openstack.placement import util - - -# TODO(cdent): NAME points to the config project being used, so for -# now this is "nova" but we probably want "placement" eventually. -NAME = "nova" - - -def deploy(conf): - """Assemble the middleware pipeline leading to the placement app.""" - if conf.api.auth_strategy == 'noauth2': - auth_middleware = auth.NoAuthMiddleware - else: - # Do not use 'oslo_config_project' param here as the conf - # location may have been overridden earlier in the deployment - # process with OS_PLACEMENT_CONFIG_DIR in wsgi.py. - auth_middleware = auth.filter_factory( - {}, oslo_config_config=conf) - - # Pass in our CORS config, if any, manually as that's a) - # explicit, b) makes testing more straightfoward, c) let's - # us control the use of cors by the presence of its config. - conf.register_opts(cors.CORS_OPTS, 'cors') - if conf.cors.allowed_origin: - cors_middleware = oslo_middleware.CORS.factory( - {}, **conf.cors) - else: - cors_middleware = None - - context_middleware = auth.PlacementKeystoneContext - req_id_middleware = oslo_middleware.RequestId - microversion_middleware = mp_middleware.MicroversionMiddleware - fault_middleware = fault_wrap.FaultWrapper - request_log = requestlog.RequestLog - - application = handler.PlacementHandler() - # configure microversion middleware in the old school way - application = microversion_middleware( - application, microversion.SERVICE_TYPE, microversion.VERSIONS, - json_error_formatter=util.json_error_formatter) - - # NOTE(cdent): The ordering here is important. The list is ordered - # from the inside out. For a single request req_id_middleware is called - # first and microversion_middleware last. Then the request is finally - # passed to the application (the PlacementHandler). At that point - # the response ascends the middleware in the reverse of the - # order the request went in. This order ensures that log messages - # all see the same contextual information including request id and - # authentication information. - for middleware in (fault_middleware, - request_log, - context_middleware, - auth_middleware, - cors_middleware, - req_id_middleware, - ): - if middleware: - application = middleware(application) - - # NOTE(mriedem): Ignore scope check UserWarnings from oslo.policy. - if not conf.oslo_policy.enforce_scope: - import warnings - warnings.filterwarnings('ignore', - message="Policy .* failed scope check", - category=UserWarning) - - return application - - -def update_database(): - """Do any database updates required at process boot time, such as - updating the traits table. - """ - ctx = db_api.DbContext() - resource_provider.ensure_trait_sync(ctx) - resource_provider.ensure_rc_cache(ctx) - - -# NOTE(cdent): Althought project_name is no longer used because of the -# resolution of https://bugs.launchpad.net/nova/+bug/1734491, loadapp() -# is considered a public interface for the creation of a placement -# WSGI app so must maintain its interface. The canonical placement WSGI -# app is created by init_application in wsgi.py, but this is not -# required and in fact can be limiting. loadapp() may be used from -# fixtures or arbitrary WSGI frameworks and loaders. -def loadapp(config, project_name=NAME): - """WSGI application creator for placement. - - :param config: An olso_config.cfg.ConfigOpts containing placement - configuration. - :param project_name: oslo_config project name. Ignored, preserved for - backwards compatibility - """ - application = deploy(config) - update_database() - return application diff --git a/nova/api/openstack/placement/direct.py b/nova/api/openstack/placement/direct.py deleted file mode 100644 index 66e11e7f62ce..000000000000 --- a/nova/api/openstack/placement/direct.py +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Call any URI in the placement service directly without real HTTP. - -This is useful for those cases where processes wish to manipulate the -Placement datastore but do not want to run Placement as a long running -service. A PlacementDirect context manager is provided. Within that -HTTP requests may be made as normal but they will not actually traverse -a real socket. -""" - -from keystoneauth1 import adapter -from keystoneauth1 import session -import mock -from oslo_utils import uuidutils -import requests -from wsgi_intercept import interceptor - -from nova.api.openstack.placement import deploy - - -class PlacementDirect(interceptor.RequestsInterceptor): - """Provide access to the placement service without real HTTP. - - wsgi-intercept is used to provide a keystoneauth1 Adapter that has access - to an in-process placement service. This provides access to making changes - to the placement database without requiring HTTP over the network - it - remains in-process. - - Authentication to the service is turned off; admin access is assumed. - - Access is provided via a context manager which is responsible for - turning the wsgi-intercept on and off, and setting and removing - mocks required to keystoneauth1 to work around endpoint discovery. - - Example:: - - with PlacementDirect(cfg.CONF, latest_microversion=True) as client: - allocations = client.get('/allocations/%s' % consumer) - - :param conf: An oslo config with the options used to configure - the placement service (notably database connection - string). - :param latest_microversion: If True, API requests will use the latest - microversion if not otherwise specified. If - False (the default), the base microversion is - the default. - """ - - def __init__(self, conf, latest_microversion=False): - conf.set_override('auth_strategy', 'noauth2', group='api') - app = lambda: deploy.loadapp(conf) - self.url = 'http://%s/placement' % str(uuidutils.generate_uuid()) - # Supply our own session so the wsgi-intercept can intercept - # the right thing. - request_session = requests.Session() - headers = { - 'x-auth-token': 'admin', - } - # TODO(efried): See below - if latest_microversion: - headers['OpenStack-API-Version'] = 'placement latest' - self.adapter = adapter.Adapter( - session.Session(auth=None, session=request_session, - additional_headers=headers), - service_type='placement', raise_exc=False) - # TODO(efried): Figure out why this isn't working: - # default_microversion='latest' if latest_microversion else None) - self._mocked_endpoint = mock.patch( - 'keystoneauth1.session.Session.get_endpoint', - new=mock.Mock(return_value=self.url)) - super(PlacementDirect, self).__init__(app, url=self.url) - - def __enter__(self): - """Start the wsgi-intercept interceptor and keystone endpoint mock. - - A no auth ksa Adapter is provided to the context being managed. - """ - super(PlacementDirect, self).__enter__() - self._mocked_endpoint.start() - return self.adapter - - def __exit__(self, *exc): - self._mocked_endpoint.stop() - return super(PlacementDirect, self).__exit__(*exc) diff --git a/nova/api/openstack/placement/errors.py b/nova/api/openstack/placement/errors.py deleted file mode 100644 index 15e4fbc4cddf..000000000000 --- a/nova/api/openstack/placement/errors.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Error code symbols to be used in structured JSON error responses. - -These are strings to be used in the 'code' attribute, as described by -the API guideline on `errors`_. - -There must be only one instance of any string value and it should have -only one associated constant SYMBOL. - -In a WSGI handler (representing the sole handler for an HTTP method and -URI) each error condition should get a separate error code. Reusing an -error code in a different handler is not just acceptable, but useful. - -For example 'placement.inventory.inuse' is meaningful and correct in both -``PUT /resource_providers/{uuid}/inventories`` and ``DELETE`` on the same -URI. - -.. _errors: http://specs.openstack.org/openstack/api-wg/guidelines/errors.html -""" - -# NOTE(cdent): This is the simplest thing that can possibly work, for now. -# If it turns out we want to automate this, or put different resources in -# different files, or otherwise change things, that's fine. The only thing -# that needs to be maintained as the same are the strings that API end -# users use. How they are created is completely fungible. - - -# Do not change the string values. Once set, they are set. -# Do not reuse string values. There should be only one symbol for any -# value. -DEFAULT = 'placement.undefined_code' -INVENTORY_INUSE = 'placement.inventory.inuse' -CONCURRENT_UPDATE = 'placement.concurrent_update' -DUPLICATE_NAME = 'placement.duplicate_name' -PROVIDER_IN_USE = 'placement.resource_provider.inuse' -PROVIDER_CANNOT_DELETE_PARENT = ( - 'placement.resource_provider.cannot_delete_parent') -RESOURCE_PROVIDER_NOT_FOUND = 'placement.resource_provider.not_found' diff --git a/nova/api/openstack/placement/exception.py b/nova/api/openstack/placement/exception.py deleted file mode 100644 index f6fa3ec7e4f0..000000000000 --- a/nova/api/openstack/placement/exception.py +++ /dev/null @@ -1,207 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Exceptions for use in the Placement API.""" - -# NOTE(cdent): The exceptions are copied from nova.exception, where they -# were originally used. To prepare for extracting placement to its own -# repository we wish to no longer do that. Instead, exceptions used by -# placement should be in the placement hierarchy. - -from oslo_log import log as logging - -from nova.i18n import _ - - -LOG = logging.getLogger(__name__) - - -class _BaseException(Exception): - """Base Exception - - To correctly use this class, inherit from it and define - a 'msg_fmt' property. That msg_fmt will get printf'd - with the keyword arguments provided to the constructor. - - """ - msg_fmt = _("An unknown exception occurred.") - - def __init__(self, message=None, **kwargs): - self.kwargs = kwargs - - if not message: - try: - message = self.msg_fmt % kwargs - except Exception: - # NOTE(melwitt): This is done in a separate method so it can be - # monkey-patched during testing to make it a hard failure. - self._log_exception() - message = self.msg_fmt - - self.message = message - super(_BaseException, self).__init__(message) - - def _log_exception(self): - # kwargs doesn't match a variable in the message - # log the issue and the kwargs - LOG.exception('Exception in string format operation') - for name, value in self.kwargs.items(): - LOG.error("%s: %s" % (name, value)) # noqa - - def format_message(self): - # Use the first argument to the python Exception object which - # should be our full exception message, (see __init__). - return self.args[0] - - -class NotFound(_BaseException): - msg_fmt = _("Resource could not be found.") - - -class Exists(_BaseException): - msg_fmt = _("Resource already exists.") - - -class InvalidInventory(_BaseException): - msg_fmt = _("Inventory for '%(resource_class)s' on " - "resource provider '%(resource_provider)s' invalid.") - - -class CannotDeleteParentResourceProvider(_BaseException): - msg_fmt = _("Cannot delete resource provider that is a parent of " - "another. Delete child providers first.") - - -class ConcurrentUpdateDetected(_BaseException): - msg_fmt = _("Another thread concurrently updated the data. " - "Please retry your update") - - -class ResourceProviderConcurrentUpdateDetected(ConcurrentUpdateDetected): - msg_fmt = _("Another thread concurrently updated the resource provider " - "data. Please retry your update") - - -class InvalidAllocationCapacityExceeded(InvalidInventory): - msg_fmt = _("Unable to create allocation for '%(resource_class)s' on " - "resource provider '%(resource_provider)s'. The requested " - "amount would exceed the capacity.") - - -class InvalidAllocationConstraintsViolated(InvalidInventory): - msg_fmt = _("Unable to create allocation for '%(resource_class)s' on " - "resource provider '%(resource_provider)s'. The requested " - "amount would violate inventory constraints.") - - -class InvalidInventoryCapacity(InvalidInventory): - msg_fmt = _("Invalid inventory for '%(resource_class)s' on " - "resource provider '%(resource_provider)s'. " - "The reserved value is greater than or equal to total.") - - -class InvalidInventoryCapacityReservedCanBeTotal(InvalidInventoryCapacity): - msg_fmt = _("Invalid inventory for '%(resource_class)s' on " - "resource provider '%(resource_provider)s'. " - "The reserved value is greater than total.") - - -# An exception with this name is used on both sides of the placement/ -# nova interaction. -class InventoryInUse(InvalidInventory): - # NOTE(mriedem): This message cannot change without impacting the - # nova.scheduler.client.report._RE_INV_IN_USE regex. - msg_fmt = _("Inventory for '%(resource_classes)s' on " - "resource provider '%(resource_provider)s' in use.") - - -class InventoryWithResourceClassNotFound(NotFound): - msg_fmt = _("No inventory of class %(resource_class)s found.") - - -class MaxDBRetriesExceeded(_BaseException): - msg_fmt = _("Max retries of DB transaction exceeded attempting to " - "perform %(action)s.") - - -class ObjectActionError(_BaseException): - msg_fmt = _('Object action %(action)s failed because: %(reason)s') - - -class PolicyNotAuthorized(_BaseException): - msg_fmt = _("Policy does not allow %(action)s to be performed.") - - -class ResourceClassCannotDeleteStandard(_BaseException): - msg_fmt = _("Cannot delete standard resource class %(resource_class)s.") - - -class ResourceClassCannotUpdateStandard(_BaseException): - msg_fmt = _("Cannot update standard resource class %(resource_class)s.") - - -class ResourceClassExists(_BaseException): - msg_fmt = _("Resource class %(resource_class)s already exists.") - - -class ResourceClassInUse(_BaseException): - msg_fmt = _("Cannot delete resource class %(resource_class)s. " - "Class is in use in inventory.") - - -class ResourceClassNotFound(NotFound): - msg_fmt = _("No such resource class %(resource_class)s.") - - -# An exception with this name is used on both sides of the placement/ -# nova interaction. -class ResourceProviderInUse(_BaseException): - msg_fmt = _("Resource provider has allocations.") - - -class TraitCannotDeleteStandard(_BaseException): - msg_fmt = _("Cannot delete standard trait %(name)s.") - - -class TraitExists(_BaseException): - msg_fmt = _("The Trait %(name)s already exists") - - -class TraitInUse(_BaseException): - msg_fmt = _("The trait %(name)s is in use by a resource provider.") - - -class TraitNotFound(NotFound): - msg_fmt = _("No such trait(s): %(names)s.") - - -class ProjectNotFound(NotFound): - msg_fmt = _("No such project(s): %(external_id)s.") - - -class ProjectExists(Exists): - msg_fmt = _("The project %(external_id)s already exists.") - - -class UserNotFound(NotFound): - msg_fmt = _("No such user(s): %(external_id)s.") - - -class UserExists(Exists): - msg_fmt = _("The user %(external_id)s already exists.") - - -class ConsumerNotFound(NotFound): - msg_fmt = _("No such consumer(s): %(uuid)s.") - - -class ConsumerExists(Exists): - msg_fmt = _("The consumer %(uuid)s already exists.") diff --git a/nova/api/openstack/placement/fault_wrap.py b/nova/api/openstack/placement/fault_wrap.py deleted file mode 100644 index 764d628b496c..000000000000 --- a/nova/api/openstack/placement/fault_wrap.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Simple middleware for safely catching unexpected exceptions.""" - -# NOTE(cdent): This is a super simplified replacement for the nova -# FaultWrapper, which does more than placement needs. - -from oslo_log import log as logging -import six -from webob import exc - -from nova.api.openstack.placement import util - -LOG = logging.getLogger(__name__) - - -class FaultWrapper(object): - """Turn an uncaught exception into a status 500. - - Uncaught exceptions usually shouldn't happen, if it does it - means there is a bug in the placement service, which should be - fixed. - """ - - def __init__(self, application): - self.application = application - - def __call__(self, environ, start_response): - try: - return self.application(environ, start_response) - except Exception as unexpected_exception: - LOG.exception('Placement API unexpected error: %s', - unexpected_exception) - formatted_exception = exc.HTTPInternalServerError( - six.text_type(unexpected_exception)) - formatted_exception.json_formatter = util.json_error_formatter - return formatted_exception.generate_response( - environ, start_response) diff --git a/nova/api/openstack/placement/handler.py b/nova/api/openstack/placement/handler.py deleted file mode 100644 index c714c464c5f2..000000000000 --- a/nova/api/openstack/placement/handler.py +++ /dev/null @@ -1,231 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Handlers for placement API. - -Individual handlers are associated with URL paths in the -ROUTE_DECLARATIONS dictionary. At the top level each key is a Routes -compliant path. The value of that key is a dictionary mapping -individual HTTP request methods to a Python function representing a -simple WSGI application for satisfying that request. - -The ``make_map`` method processes ROUTE_DECLARATIONS to create a -Routes.Mapper, including automatic handlers to respond with a -405 when a request is made against a valid URL with an invalid -method. -""" - -import routes -import webob - -from oslo_log import log as logging - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement.handlers import aggregate -from nova.api.openstack.placement.handlers import allocation -from nova.api.openstack.placement.handlers import allocation_candidate -from nova.api.openstack.placement.handlers import inventory -from nova.api.openstack.placement.handlers import reshaper -from nova.api.openstack.placement.handlers import resource_class -from nova.api.openstack.placement.handlers import resource_provider -from nova.api.openstack.placement.handlers import root -from nova.api.openstack.placement.handlers import trait -from nova.api.openstack.placement.handlers import usage -from nova.api.openstack.placement import util -from nova.i18n import _ - -LOG = logging.getLogger(__name__) - -# URLs and Handlers -# NOTE(cdent): When adding URLs here, do not use regex patterns in -# the path parameters (e.g. {uuid:[0-9a-zA-Z-]+}) as that will lead -# to 404s that are controlled outside of the individual resources -# and thus do not include specific information on the why of the 404. -ROUTE_DECLARATIONS = { - '/': { - 'GET': root.home, - }, - # NOTE(cdent): This allows '/placement/' and '/placement' to - # both work as the root of the service, which we probably want - # for those situations where the service is mounted under a - # prefix (as it is in devstack). While weird, an empty string is - # a legit key in a dictionary and matches as desired in Routes. - '': { - 'GET': root.home, - }, - '/resource_classes': { - 'GET': resource_class.list_resource_classes, - 'POST': resource_class.create_resource_class - }, - '/resource_classes/{name}': { - 'GET': resource_class.get_resource_class, - 'PUT': resource_class.update_resource_class, - 'DELETE': resource_class.delete_resource_class, - }, - '/resource_providers': { - 'GET': resource_provider.list_resource_providers, - 'POST': resource_provider.create_resource_provider - }, - '/resource_providers/{uuid}': { - 'GET': resource_provider.get_resource_provider, - 'DELETE': resource_provider.delete_resource_provider, - 'PUT': resource_provider.update_resource_provider - }, - '/resource_providers/{uuid}/inventories': { - 'GET': inventory.get_inventories, - 'POST': inventory.create_inventory, - 'PUT': inventory.set_inventories, - 'DELETE': inventory.delete_inventories - }, - '/resource_providers/{uuid}/inventories/{resource_class}': { - 'GET': inventory.get_inventory, - 'PUT': inventory.update_inventory, - 'DELETE': inventory.delete_inventory - }, - '/resource_providers/{uuid}/usages': { - 'GET': usage.list_usages - }, - '/resource_providers/{uuid}/aggregates': { - 'GET': aggregate.get_aggregates, - 'PUT': aggregate.set_aggregates - }, - '/resource_providers/{uuid}/allocations': { - 'GET': allocation.list_for_resource_provider, - }, - '/allocations': { - 'POST': allocation.set_allocations, - }, - '/allocations/{consumer_uuid}': { - 'GET': allocation.list_for_consumer, - 'PUT': allocation.set_allocations_for_consumer, - 'DELETE': allocation.delete_allocations, - }, - '/allocation_candidates': { - 'GET': allocation_candidate.list_allocation_candidates, - }, - '/traits': { - 'GET': trait.list_traits, - }, - '/traits/{name}': { - 'GET': trait.get_trait, - 'PUT': trait.put_trait, - 'DELETE': trait.delete_trait, - }, - '/resource_providers/{uuid}/traits': { - 'GET': trait.list_traits_for_resource_provider, - 'PUT': trait.update_traits_for_resource_provider, - 'DELETE': trait.delete_traits_for_resource_provider - }, - '/usages': { - 'GET': usage.get_total_usages, - }, - '/reshaper': { - 'POST': reshaper.reshape, - }, -} - - -def dispatch(environ, start_response, mapper): - """Find a matching route for the current request. - - If no match is found, raise a 404 response. - If there is a matching route, but no matching handler - for the given method, raise a 405. - """ - result = mapper.match(environ=environ) - if result is None: - raise webob.exc.HTTPNotFound( - json_formatter=util.json_error_formatter) - # We can't reach this code without action being present. - handler = result.pop('action') - environ['wsgiorg.routing_args'] = ((), result) - return handler(environ, start_response) - - -def handle_405(environ, start_response): - """Return a 405 response when method is not allowed. - - If _methods are in routing_args, send an allow header listing - the methods that are possible on the provided URL. - """ - _methods = util.wsgi_path_item(environ, '_methods') - headers = {} - if _methods: - # Ensure allow header is a python 2 or 3 native string (thus - # not unicode in python 2 but stay a string in python 3) - # In the process done by Routes to save the allowed methods - # to its routing table they become unicode in py2. - headers['allow'] = str(_methods) - # Use Exception class as WSGI Application. We don't want to raise here. - response = webob.exc.HTTPMethodNotAllowed( - _('The method specified is not allowed for this resource.'), - headers=headers, json_formatter=util.json_error_formatter) - return response(environ, start_response) - - -def make_map(declarations): - """Process route declarations to create a Route Mapper.""" - mapper = routes.Mapper() - for route, targets in declarations.items(): - allowed_methods = [] - for method in targets: - mapper.connect(route, action=targets[method], - conditions=dict(method=[method])) - allowed_methods.append(method) - allowed_methods = ', '.join(allowed_methods) - mapper.connect(route, action=handle_405, _methods=allowed_methods) - return mapper - - -class PlacementHandler(object): - """Serve Placement API. - - Dispatch to handlers defined in ROUTE_DECLARATIONS. - """ - - def __init__(self, **local_config): - # NOTE(cdent): Local config currently unused. - self._map = make_map(ROUTE_DECLARATIONS) - - def __call__(self, environ, start_response): - # Check that an incoming request with a content-length header - # that is an integer > 0 and not empty, also has a content-type - # header that is not empty. If not raise a 400. - clen = environ.get('CONTENT_LENGTH') - try: - if clen and (int(clen) > 0) and not environ.get('CONTENT_TYPE'): - raise webob.exc.HTTPBadRequest( - _('content-type header required when content-length > 0'), - json_formatter=util.json_error_formatter) - except ValueError as exc: - raise webob.exc.HTTPBadRequest( - _('content-length header must be an integer'), - json_formatter=util.json_error_formatter) - try: - return dispatch(environ, start_response, self._map) - # Trap the NotFound exceptions raised by the objects used - # with the API and transform them into webob.exc.HTTPNotFound. - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - exc, json_formatter=util.json_error_formatter) - except exception.PolicyNotAuthorized as exc: - raise webob.exc.HTTPForbidden( - exc.format_message(), - json_formatter=util.json_error_formatter) - # Remaining uncaught exceptions will rise first to the Microversion - # middleware, where any WebOb generated exceptions will be caught and - # transformed into legit HTTP error responses (with microversion - # headers added), and then to the FaultWrapper middleware which will - # catch anything else and transform them into 500 responses. - # NOTE(cdent): There should be very few uncaught exceptions which are - # not WebOb exceptions at this stage as the handlers are contained by - # the wsgify decorator which will transform those exceptions to - # responses itself. diff --git a/nova/api/openstack/placement/handlers/__init__.py b/nova/api/openstack/placement/handlers/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/api/openstack/placement/handlers/aggregate.py b/nova/api/openstack/placement/handlers/aggregate.py deleted file mode 100644 index a26839c3739d..000000000000 --- a/nova/api/openstack/placement/handlers/aggregate.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Aggregate handlers for Placement API.""" - -from oslo_db import exception as db_exc -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import aggregate as policies -from nova.api.openstack.placement.schemas import aggregate as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -_INCLUDE_GENERATION_VERSION = (1, 19) - - -def _send_aggregates(req, resource_provider, aggregate_uuids): - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - response = req.response - response.status = 200 - payload = _serialize_aggregates(aggregate_uuids) - if want_version.matches(min_version=_INCLUDE_GENERATION_VERSION): - payload['resource_provider_generation'] = resource_provider.generation - response.body = encodeutils.to_utf8( - jsonutils.dumps(payload)) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - req.response.cache_control = 'no-cache' - # We never get an aggregate itself, we get the list of aggregates - # that are associated with a resource provider. We don't record the - # time when that association was made and the time when an aggregate - # uuid was created is not relevant, so here we punt and use utcnow. - req.response.last_modified = timeutils.utcnow(with_timezone=True) - return response - - -def _serialize_aggregates(aggregate_uuids): - return {'aggregates': aggregate_uuids} - - -def _set_aggregates(resource_provider, aggregate_uuids, - increment_generation=False): - """Set aggregates for the resource provider. - - If increment generation is true, the resource provider generation - will be incremented if possible. If that fails (because something - else incremented the generation in another thread), a - ConcurrentUpdateDetected will be raised. - """ - # NOTE(cdent): It's not clear what the DBDuplicateEntry handling - # is doing here, set_aggregates already handles that, but I'm leaving - # it here because it was already there. - try: - resource_provider.set_aggregates( - aggregate_uuids, increment_generation=increment_generation) - except exception.ConcurrentUpdateDetected as exc: - raise webob.exc.HTTPConflict( - _('Update conflict: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - except db_exc.DBDuplicateEntry as exc: - raise webob.exc.HTTPConflict( - _('Update conflict: %(error)s') % {'error': exc}) - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -@microversion.version_handler('1.1') -def get_aggregates(req): - """GET a list of aggregates associated with a resource provider. - - If the resource provider does not exist return a 404. - - On success return a 200 with an application/json body containing a - list of aggregate uuids. - """ - context = req.environ['placement.context'] - context.can(policies.LIST) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - aggregate_uuids = resource_provider.get_aggregates() - - return _send_aggregates(req, resource_provider, aggregate_uuids) - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -@microversion.version_handler('1.1') -def set_aggregates(req): - context = req.environ['placement.context'] - context.can(policies.UPDATE) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - consider_generation = want_version.matches( - min_version=_INCLUDE_GENERATION_VERSION) - put_schema = schema.PUT_AGGREGATES_SCHEMA_V1_1 - if consider_generation: - put_schema = schema.PUT_AGGREGATES_SCHEMA_V1_19 - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - data = util.extract_json(req.body, put_schema) - if consider_generation: - # Check for generation conflict - rp_gen = data['resource_provider_generation'] - if resource_provider.generation != rp_gen: - raise webob.exc.HTTPConflict( - _("Resource provider's generation already changed. Please " - "update the generation and try again."), - comment=errors.CONCURRENT_UPDATE) - aggregate_uuids = data['aggregates'] - else: - aggregate_uuids = data - _set_aggregates(resource_provider, aggregate_uuids, - increment_generation=consider_generation) - - return _send_aggregates(req, resource_provider, aggregate_uuids) diff --git a/nova/api/openstack/placement/handlers/allocation.py b/nova/api/openstack/placement/handlers/allocation.py deleted file mode 100644 index 9b2f5d8f3ae1..000000000000 --- a/nova/api/openstack/placement/handlers/allocation.py +++ /dev/null @@ -1,576 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API handlers for setting and deleting allocations.""" - -import collections -import uuid - -from oslo_log import log as logging -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import excutils -from oslo_utils import timeutils -from oslo_utils import uuidutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import allocation as policies -from nova.api.openstack.placement.schemas import allocation as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -LOG = logging.getLogger(__name__) - - -def _last_modified_from_allocations(allocations, want_version): - """Given a set of allocation objects, returns the last modified timestamp. - """ - # NOTE(cdent): The last_modified for an allocation will always be - # based off the created_at column because allocations are only - # ever inserted, never updated. - last_modified = None - # Only calculate last-modified if we are using a microversion that - # supports it. - get_last_modified = want_version and want_version.matches((1, 15)) - for allocation in allocations: - if get_last_modified: - last_modified = util.pick_last_modified(last_modified, allocation) - - last_modified = last_modified or timeutils.utcnow(with_timezone=True) - return last_modified - - -def _serialize_allocations_for_consumer(allocations, want_version): - """Turn a list of allocations into a dict by resource provider uuid. - - { - 'allocations': { - RP_UUID_1: { - 'generation': GENERATION, - 'resources': { - 'DISK_GB': 4, - 'VCPU': 2 - } - }, - RP_UUID_2: { - 'generation': GENERATION, - 'resources': { - 'DISK_GB': 6, - 'VCPU': 3 - } - } - }, - # project_id and user_id are added with microverion 1.12 - 'project_id': PROJECT_ID, - 'user_id': USER_ID, - # Generation for consumer >= 1.28 - 'consumer_generation': 1 - } - """ - allocation_data = collections.defaultdict(dict) - for allocation in allocations: - key = allocation.resource_provider.uuid - if 'resources' not in allocation_data[key]: - allocation_data[key]['resources'] = {} - - resource_class = allocation.resource_class - allocation_data[key]['resources'][resource_class] = allocation.used - generation = allocation.resource_provider.generation - allocation_data[key]['generation'] = generation - - result = {'allocations': allocation_data} - if allocations and want_version.matches((1, 12)): - # We're looking at a list of allocations by consumer id so project and - # user are consistent across the list - consumer = allocations[0].consumer - project_id = consumer.project.external_id - user_id = consumer.user.external_id - result['project_id'] = project_id - result['user_id'] = user_id - show_consumer_gen = want_version.matches((1, 28)) - if show_consumer_gen: - result['consumer_generation'] = consumer.generation - - return result - - -def _serialize_allocations_for_resource_provider(allocations, - resource_provider, - want_version): - """Turn a list of allocations into a dict by consumer id. - - {'resource_provider_generation': GENERATION, - 'allocations': - CONSUMER_ID_1: { - 'resources': { - 'DISK_GB': 4, - 'VCPU': 2 - }, - # Generation for consumer >= 1.28 - 'consumer_generation': 0 - }, - CONSUMER_ID_2: { - 'resources': { - 'DISK_GB': 6, - 'VCPU': 3 - }, - # Generation for consumer >= 1.28 - 'consumer_generation': 0 - } - } - """ - show_consumer_gen = want_version.matches((1, 28)) - allocation_data = collections.defaultdict(dict) - for allocation in allocations: - key = allocation.consumer.uuid - if 'resources' not in allocation_data[key]: - allocation_data[key]['resources'] = {} - - resource_class = allocation.resource_class - allocation_data[key]['resources'][resource_class] = allocation.used - - if show_consumer_gen: - consumer_gen = None - if allocation.consumer is not None: - consumer_gen = allocation.consumer.generation - allocation_data[key]['consumer_generation'] = consumer_gen - - result = {'allocations': allocation_data} - result['resource_provider_generation'] = resource_provider.generation - return result - - -# TODO(cdent): Extracting this is useful, for reuse by reshaper code, -# but having it in this file seems wrong, however, since it uses -# _new_allocations it's being left here for now. We need a place for shared -# handler code, but util.py is already too big and too diverse. -def create_allocation_list(context, data, consumers): - """Create an AllocationList based on provided data. - - :param context: The placement context. - :param data: A dictionary of multiple allocations by consumer uuid. - :param consumers: A dictionary, keyed by consumer UUID, of Consumer objects - :return: An AllocationList. - :raises: `webob.exc.HTTPBadRequest` if a resource provider included in the - allocations does not exist. - """ - allocation_objects = [] - - for consumer_uuid in data: - allocations = data[consumer_uuid]['allocations'] - consumer = consumers[consumer_uuid] - if allocations: - rp_objs = _resource_providers_by_uuid(context, allocations.keys()) - for resource_provider_uuid in allocations: - resource_provider = rp_objs[resource_provider_uuid] - resources = allocations[resource_provider_uuid]['resources'] - new_allocations = _new_allocations(context, - resource_provider, - consumer, - resources) - allocation_objects.extend(new_allocations) - else: - # The allocations are empty, which means wipe them out. - # Internal to the allocation object this is signalled by a - # used value of 0. - allocations = rp_obj.AllocationList.get_all_by_consumer_id( - context, consumer_uuid) - for allocation in allocations: - allocation.used = 0 - allocation_objects.append(allocation) - - return rp_obj.AllocationList(context, objects=allocation_objects) - - -def inspect_consumers(context, data, want_version): - """Look at consumer data in allocations and create consumers as needed. - - Keep a record of the consumers that are created in case they need - to be removed later. - - If an exception is raised by ensure_consumer, commonly HTTPConflict but - also anything else, the newly created consumers will be deleted and the - exception reraised to the caller. - - :param context: The placement context. - :param data: A dictionary of multiple allocations by consumer uuid. - :param want_version: the microversion matcher. - :return: A tuple of a dict of all consumer objects (by consumer uuid) - and a list of those consumer objects which are new. - """ - # First, ensure that all consumers referenced in the payload actually - # exist. And if not, create them. Keep a record of auto-created consumers - # so we can clean them up if the end allocation replace_all() fails. - consumers = {} # dict of Consumer objects, keyed by consumer UUID - new_consumers_created = [] - for consumer_uuid in data: - project_id = data[consumer_uuid]['project_id'] - user_id = data[consumer_uuid]['user_id'] - consumer_generation = data[consumer_uuid].get('consumer_generation') - try: - consumer, new_consumer_created = util.ensure_consumer( - context, consumer_uuid, project_id, user_id, - consumer_generation, want_version) - if new_consumer_created: - new_consumers_created.append(consumer) - consumers[consumer_uuid] = consumer - except Exception: - # If any errors (for instance, a consumer generation conflict) - # occur when ensuring consumer records above, make sure we delete - # any auto-created consumers. - with excutils.save_and_reraise_exception(): - delete_consumers(new_consumers_created) - return consumers, new_consumers_created - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def list_for_consumer(req): - """List allocations associated with a consumer.""" - context = req.environ['placement.context'] - context.can(policies.ALLOC_LIST) - consumer_id = util.wsgi_path_item(req.environ, 'consumer_uuid') - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - - # NOTE(cdent): There is no way for a 404 to be returned here, - # only an empty result. We do not have a way to validate a - # consumer id. - allocations = rp_obj.AllocationList.get_all_by_consumer_id( - context, consumer_id) - - output = _serialize_allocations_for_consumer(allocations, want_version) - last_modified = _last_modified_from_allocations(allocations, want_version) - allocations_json = jsonutils.dumps(output) - - response = req.response - response.status = 200 - response.body = encodeutils.to_utf8(allocations_json) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.last_modified = last_modified - response.cache_control = 'no-cache' - return response - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def list_for_resource_provider(req): - """List allocations associated with a resource provider.""" - # TODO(cdent): On a shared resource provider (for example a - # giant disk farm) this list could get very long. At the moment - # we have no facility for limiting the output. Given that we are - # using a dict of dicts for the output we are potentially limiting - # ourselves in terms of sorting and filtering. - context = req.environ['placement.context'] - context.can(policies.RP_ALLOC_LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - uuid = util.wsgi_path_item(req.environ, 'uuid') - - # confirm existence of resource provider so we get a reasonable - # 404 instead of empty list - try: - rp = rp_obj.ResourceProvider.get_by_uuid(context, uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("Resource provider '%(rp_uuid)s' not found: %(error)s") % - {'rp_uuid': uuid, 'error': exc}) - - allocs = rp_obj.AllocationList.get_all_by_resource_provider(context, rp) - - output = _serialize_allocations_for_resource_provider( - allocs, rp, want_version) - last_modified = _last_modified_from_allocations(allocs, want_version) - allocations_json = jsonutils.dumps(output) - - response = req.response - response.status = 200 - response.body = encodeutils.to_utf8(allocations_json) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.last_modified = last_modified - response.cache_control = 'no-cache' - return response - - -def _resource_providers_by_uuid(ctx, rp_uuids): - """Helper method that returns a dict, keyed by resource provider UUID, of - ResourceProvider objects. - - :param ctx: The placement context. - :param rp_uuids: iterable of UUIDs for providers to fetch. - :raises: `webob.exc.HTTPBadRequest` if any of the UUIDs do not refer to - an existing resource provider. - """ - res = {} - for rp_uuid in rp_uuids: - # TODO(jaypipes): Clearly, this is not efficient to do one query for - # each resource provider UUID in the allocations instead of doing a - # single query for all the UUIDs. However, since - # ResourceProviderList.get_all_by_filters() is way too complicated for - # this purpose and doesn't raise NotFound anyway, we'll do this. - # Perhaps consider adding a ResourceProviderList.get_all_by_uuids() - # later on? - try: - res[rp_uuid] = rp_obj.ResourceProvider.get_by_uuid(ctx, rp_uuid) - except exception.NotFound: - raise webob.exc.HTTPBadRequest( - _("Allocation for resource provider '%(rp_uuid)s' " - "that does not exist.") % - {'rp_uuid': rp_uuid}) - return res - - -def _new_allocations(context, resource_provider, consumer, resources): - """Create new allocation objects for a set of resources - - Returns a list of Allocation objects - - :param context: The placement context. - :param resource_provider: The resource provider that has the resources. - :param consumer: The Consumer object consuming the resources. - :param resources: A dict of resource classes and values. - """ - allocations = [] - for resource_class in resources: - allocation = rp_obj.Allocation( - resource_provider=resource_provider, - consumer=consumer, - resource_class=resource_class, - used=resources[resource_class]) - allocations.append(allocation) - return allocations - - -def delete_consumers(consumers): - """Helper function that deletes any consumer object supplied to it - - :param consumers: iterable of Consumer objects to delete - """ - for consumer in consumers: - try: - consumer.delete() - LOG.debug("Deleted auto-created consumer with consumer UUID " - "%s after failed allocation", consumer.uuid) - except Exception as err: - LOG.warning("Got an exception when deleting auto-created " - "consumer with UUID %s: %s", consumer.uuid, err) - - -def _set_allocations_for_consumer(req, schema): - context = req.environ['placement.context'] - context.can(policies.ALLOC_UPDATE) - consumer_uuid = util.wsgi_path_item(req.environ, 'consumer_uuid') - if not uuidutils.is_uuid_like(consumer_uuid): - raise webob.exc.HTTPBadRequest( - _('Malformed consumer_uuid: %(consumer_uuid)s') % - {'consumer_uuid': consumer_uuid}) - consumer_uuid = str(uuid.UUID(consumer_uuid)) - data = util.extract_json(req.body, schema) - allocation_data = data['allocations'] - - # Normalize allocation data to dict. - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - if not want_version.matches((1, 12)): - allocations_dict = {} - # Allocation are list-ish, transform to dict-ish - for allocation in allocation_data: - resource_provider_uuid = allocation['resource_provider']['uuid'] - allocations_dict[resource_provider_uuid] = { - 'resources': allocation['resources'] - } - allocation_data = allocations_dict - - allocation_objects = [] - # Consumer object saved in case we need to delete the auto-created consumer - # record - consumer = None - # Whether we created a new consumer record - created_new_consumer = False - if not allocation_data: - # The allocations are empty, which means wipe them out. Internal - # to the allocation object this is signalled by a used value of 0. - # We still need to verify the consumer's generation, though, which - # we do in _ensure_consumer() - # NOTE(jaypipes): This will only occur 1.28+. The JSONSchema will - # prevent an empty allocations object from being passed when there is - # no consumer generation, so this is safe to do. - util.ensure_consumer(context, consumer_uuid, data.get('project_id'), - data.get('user_id'), data.get('consumer_generation'), - want_version) - allocations = rp_obj.AllocationList.get_all_by_consumer_id( - context, consumer_uuid) - for allocation in allocations: - allocation.used = 0 - allocation_objects.append(allocation) - else: - # If the body includes an allocation for a resource provider - # that does not exist, raise a 400. - rp_objs = _resource_providers_by_uuid(context, allocation_data.keys()) - consumer, created_new_consumer = util.ensure_consumer( - context, consumer_uuid, data.get('project_id'), - data.get('user_id'), data.get('consumer_generation'), - want_version) - for resource_provider_uuid, allocation in allocation_data.items(): - resource_provider = rp_objs[resource_provider_uuid] - new_allocations = _new_allocations(context, - resource_provider, - consumer, - allocation['resources']) - allocation_objects.extend(new_allocations) - - allocations = rp_obj.AllocationList( - context, objects=allocation_objects) - - def _create_allocations(alloc_list): - try: - alloc_list.replace_all() - LOG.debug("Successfully wrote allocations %s", alloc_list) - except Exception: - if created_new_consumer: - delete_consumers([consumer]) - raise - - try: - _create_allocations(allocations) - # InvalidInventory is a parent for several exceptions that - # indicate either that Inventory is not present, or that - # capacity limits have been exceeded. - except exception.NotFound as exc: - raise webob.exc.HTTPBadRequest( - _("Unable to allocate inventory for consumer " - "%(consumer_uuid)s: %(error)s") % - {'consumer_uuid': consumer_uuid, 'error': exc}) - except exception.InvalidInventory as exc: - raise webob.exc.HTTPConflict( - _('Unable to allocate inventory: %(error)s') % {'error': exc}) - except exception.ConcurrentUpdateDetected as exc: - raise webob.exc.HTTPConflict( - _('Inventory and/or allocations changed while attempting to ' - 'allocate: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - - req.response.status = 204 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.0', '1.7') -@util.require_content('application/json') -def set_allocations_for_consumer(req): - return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA) - - -@wsgi_wrapper.PlacementWsgify # noqa -@microversion.version_handler('1.8', '1.11') -@util.require_content('application/json') -def set_allocations_for_consumer(req): - return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_8) - - -@wsgi_wrapper.PlacementWsgify # noqa -@microversion.version_handler('1.12', '1.27') -@util.require_content('application/json') -def set_allocations_for_consumer(req): - return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_12) - - -@wsgi_wrapper.PlacementWsgify # noqa -@microversion.version_handler('1.28') -@util.require_content('application/json') -def set_allocations_for_consumer(req): - return _set_allocations_for_consumer(req, schema.ALLOCATION_SCHEMA_V1_28) - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.13') -@util.require_content('application/json') -def set_allocations(req): - context = req.environ['placement.context'] - context.can(policies.ALLOC_MANAGE) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - want_schema = schema.POST_ALLOCATIONS_V1_13 - if want_version.matches((1, 28)): - want_schema = schema.POST_ALLOCATIONS_V1_28 - data = util.extract_json(req.body, want_schema) - - consumers, new_consumers_created = inspect_consumers( - context, data, want_version) - # Create a sequence of allocation objects to be used in one - # AllocationList.replace_all() call, which will mean all the changes - # happen within a single transaction and with resource provider - # and consumer generations (if applicable) check all in one go. - allocations = create_allocation_list(context, data, consumers) - - def _create_allocations(alloc_list): - try: - alloc_list.replace_all() - LOG.debug("Successfully wrote allocations %s", alloc_list) - except Exception: - delete_consumers(new_consumers_created) - raise - - try: - _create_allocations(allocations) - except exception.NotFound as exc: - raise webob.exc.HTTPBadRequest( - _("Unable to allocate inventory %(error)s") % {'error': exc}) - except exception.InvalidInventory as exc: - # InvalidInventory is a parent for several exceptions that - # indicate either that Inventory is not present, or that - # capacity limits have been exceeded. - raise webob.exc.HTTPConflict( - _('Unable to allocate inventory: %(error)s') % {'error': exc}) - except exception.ConcurrentUpdateDetected as exc: - raise webob.exc.HTTPConflict( - _('Inventory and/or allocations changed while attempting to ' - 'allocate: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - - req.response.status = 204 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -def delete_allocations(req): - context = req.environ['placement.context'] - context.can(policies.ALLOC_DELETE) - consumer_uuid = util.wsgi_path_item(req.environ, 'consumer_uuid') - - allocations = rp_obj.AllocationList.get_all_by_consumer_id( - context, consumer_uuid) - if allocations: - try: - allocations.delete_all() - # NOTE(pumaranikar): Following NotFound exception added in the case - # when allocation is deleted from allocations list by some other - # activity. In that case, delete_all() will throw a NotFound exception. - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("Allocation for consumer with id %(id)s not found." - "error: %(error)s") % - {'id': consumer_uuid, 'error': exc}) - else: - raise webob.exc.HTTPNotFound( - _("No allocations for consumer '%(consumer_uuid)s'") % - {'consumer_uuid': consumer_uuid}) - LOG.debug("Successfully deleted allocations %s", allocations) - - req.response.status = 204 - req.response.content_type = None - return req.response diff --git a/nova/api/openstack/placement/handlers/allocation_candidate.py b/nova/api/openstack/placement/handlers/allocation_candidate.py deleted file mode 100644 index f5425cdf4ffe..000000000000 --- a/nova/api/openstack/placement/handlers/allocation_candidate.py +++ /dev/null @@ -1,332 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Placement API handlers for getting allocation candidates.""" - -import collections - -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -import six -import webob - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import allocation_candidate as \ - policies -from nova.api.openstack.placement.schemas import allocation_candidate as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -def _transform_allocation_requests_dict(alloc_reqs): - """Turn supplied list of AllocationRequest objects into a list of - allocations dicts keyed by resource provider uuid of resources involved - in the allocation request. The returned results are intended to be used - as the body of a PUT /allocations/{consumer_uuid} HTTP request at - micoversion 1.12 (and beyond). The JSON objects look like the following: - - [ - { - "allocations": { - $rp_uuid1: { - "resources": { - "MEMORY_MB": 512 - ... - } - }, - $rp_uuid2: { - "resources": { - "DISK_GB": 1024 - ... - } - } - }, - }, - ... - ] - """ - results = [] - - for ar in alloc_reqs: - # A default dict of {$rp_uuid: "resources": {}) - rp_resources = collections.defaultdict(lambda: dict(resources={})) - for rr in ar.resource_requests: - res_dict = rp_resources[rr.resource_provider.uuid]['resources'] - res_dict[rr.resource_class] = rr.amount - results.append(dict(allocations=rp_resources)) - - return results - - -def _transform_allocation_requests_list(alloc_reqs): - """Turn supplied list of AllocationRequest objects into a list of dicts of - resources involved in the allocation request. The returned results is - intended to be able to be used as the body of a PUT - /allocations/{consumer_uuid} HTTP request, prior to microversion 1.12, - so therefore we return a list of JSON objects that looks like the - following: - - [ - { - "allocations": [ - { - "resource_provider": { - "uuid": $rp_uuid, - } - "resources": { - $resource_class: $requested_amount, ... - }, - }, ... - ], - }, ... - ] - """ - results = [] - for ar in alloc_reqs: - provider_resources = collections.defaultdict(dict) - for rr in ar.resource_requests: - res_dict = provider_resources[rr.resource_provider.uuid] - res_dict[rr.resource_class] = rr.amount - - allocs = [ - { - "resource_provider": { - "uuid": rp_uuid, - }, - "resources": resources, - } for rp_uuid, resources in provider_resources.items() - ] - alloc = { - "allocations": allocs - } - results.append(alloc) - return results - - -def _transform_provider_summaries(p_sums, requests, want_version): - """Turn supplied list of ProviderSummary objects into a dict, keyed by - resource provider UUID, of dicts of provider and inventory information. - The traits only show up when `want_version` is 1.17 or newer. All the - resource classes are shown when `want_version` is 1.27 or newer while - only requested resources are included in the `provider_summaries` - for older versions. The parent and root provider uuids only show up - when `want_version` is 1.29 or newer. - - { - RP_UUID_1: { - 'resources': { - 'DISK_GB': { - 'capacity': 100, - 'used': 0, - }, - 'VCPU': { - 'capacity': 4, - 'used': 0, - } - }, - # traits shows up from microversion 1.17 - 'traits': [ - 'HW_CPU_X86_AVX512F', - 'HW_CPU_X86_AVX512CD' - ] - # parent/root provider uuids show up from microversion 1.29 - parent_provider_uuid: null, - root_provider_uuid: RP_UUID_1 - }, - RP_UUID_2: { - 'resources': { - 'DISK_GB': { - 'capacity': 100, - 'used': 0, - }, - 'VCPU': { - 'capacity': 4, - 'used': 0, - } - }, - # traits shows up from microversion 1.17 - 'traits': [ - 'HW_NIC_OFFLOAD_TSO', - 'HW_NIC_OFFLOAD_GRO' - ], - # parent/root provider uuids show up from microversion 1.29 - parent_provider_uuid: null, - root_provider_uuid: RP_UUID_2 - } - } - """ - include_traits = want_version.matches((1, 17)) - include_all_resources = want_version.matches((1, 27)) - enable_nested_providers = want_version.matches((1, 29)) - - ret = {} - requested_resources = set() - - for requested_group in requests.values(): - requested_resources |= set(requested_group.resources) - - # if include_all_resources is false, only requested resources are - # included in the provider_summaries. - for ps in p_sums: - resources = { - psr.resource_class: { - 'capacity': psr.capacity, - 'used': psr.used, - } for psr in ps.resources if ( - include_all_resources or - psr.resource_class in requested_resources) - } - - ret[ps.resource_provider.uuid] = {'resources': resources} - - if include_traits: - ret[ps.resource_provider.uuid]['traits'] = [ - t.name for t in ps.traits] - - if enable_nested_providers: - ret[ps.resource_provider.uuid]['parent_provider_uuid'] = ( - ps.resource_provider.parent_provider_uuid) - ret[ps.resource_provider.uuid]['root_provider_uuid'] = ( - ps.resource_provider.root_provider_uuid) - - return ret - - -def _exclude_nested_providers(alloc_cands): - """Exclude allocation requests and provider summaries for old microversions - if they involve more than one provider from the same tree. - """ - # Build a temporary dict, keyed by root RP UUID of sets of UUIDs of all RPs - # in that tree. - tree_rps_by_root = collections.defaultdict(set) - for ps in alloc_cands.provider_summaries: - rp_uuid = ps.resource_provider.uuid - root_uuid = ps.resource_provider.root_provider_uuid - tree_rps_by_root[root_uuid].add(rp_uuid) - # We use this to get a list of sets of providers in each tree - tree_sets = list(tree_rps_by_root.values()) - - for a_req in alloc_cands.allocation_requests[:]: - alloc_rp_uuids = set([ - arr.resource_provider.uuid for arr in a_req.resource_requests]) - # If more than one allocation is provided by the same tree, kill - # that allocation request. - if any(len(tree_set & alloc_rp_uuids) > 1 for tree_set in tree_sets): - alloc_cands.allocation_requests.remove(a_req) - - # Exclude eliminated providers from the provider summaries. - all_rp_uuids = set() - for a_req in alloc_cands.allocation_requests: - all_rp_uuids |= set( - arr.resource_provider.uuid for arr in a_req.resource_requests) - for ps in alloc_cands.provider_summaries[:]: - if ps.resource_provider.uuid not in all_rp_uuids: - alloc_cands.provider_summaries.remove(ps) - - return alloc_cands - - -def _transform_allocation_candidates(alloc_cands, requests, want_version): - """Turn supplied AllocationCandidates object into a dict containing - allocation requests and provider summaries. - - { - 'allocation_requests': , - 'provider_summaries': , - } - """ - # exclude nested providers with old microversions - if not want_version.matches((1, 29)): - alloc_cands = _exclude_nested_providers(alloc_cands) - - if want_version.matches((1, 12)): - a_reqs = _transform_allocation_requests_dict( - alloc_cands.allocation_requests) - else: - a_reqs = _transform_allocation_requests_list( - alloc_cands.allocation_requests) - - p_sums = _transform_provider_summaries( - alloc_cands.provider_summaries, requests, want_version) - - return { - 'allocation_requests': a_reqs, - 'provider_summaries': p_sums, - } - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.10') -@util.check_accept('application/json') -def list_allocation_candidates(req): - """GET a JSON object with a list of allocation requests and a JSON object - of provider summary objects - - On success return a 200 and an application/json body representing - a collection of allocation requests and provider summaries - """ - context = req.environ['placement.context'] - context.can(policies.LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - get_schema = schema.GET_SCHEMA_1_10 - if want_version.matches((1, 25)): - get_schema = schema.GET_SCHEMA_1_25 - elif want_version.matches((1, 21)): - get_schema = schema.GET_SCHEMA_1_21 - elif want_version.matches((1, 17)): - get_schema = schema.GET_SCHEMA_1_17 - elif want_version.matches((1, 16)): - get_schema = schema.GET_SCHEMA_1_16 - util.validate_query_params(req, get_schema) - - requests = util.parse_qs_request_groups(req) - limit = req.GET.getall('limit') - # JSONschema has already confirmed that limit has the form - # of an integer. - if limit: - limit = int(limit[0]) - - group_policy = req.GET.getall('group_policy') or None - # Schema ensures we get either "none" or "isolate" - if group_policy: - group_policy = group_policy[0] - else: - # group_policy is required if more than one numbered request group was - # specified. - if len([rg for rg in requests.values() if rg.use_same_provider]) > 1: - raise webob.exc.HTTPBadRequest( - _('The "group_policy" parameter is required when specifying ' - 'more than one "resources{N}" parameter.')) - - try: - cands = rp_obj.AllocationCandidates.get_by_requests( - context, requests, limit=limit, group_policy=group_policy) - except exception.ResourceClassNotFound as exc: - raise webob.exc.HTTPBadRequest( - _('Invalid resource class in resources parameter: %(error)s') % - {'error': exc}) - except exception.TraitNotFound as exc: - raise webob.exc.HTTPBadRequest(six.text_type(exc)) - - response = req.response - trx_cands = _transform_allocation_candidates(cands, requests, want_version) - json_data = jsonutils.dumps(trx_cands) - response.body = encodeutils.to_utf8(json_data) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.cache_control = 'no-cache' - response.last_modified = timeutils.utcnow(with_timezone=True) - return response diff --git a/nova/api/openstack/placement/handlers/inventory.py b/nova/api/openstack/placement/handlers/inventory.py deleted file mode 100644 index 019ada01aa30..000000000000 --- a/nova/api/openstack/placement/handlers/inventory.py +++ /dev/null @@ -1,467 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Inventory handlers for Placement API.""" - -import copy -import operator - -from oslo_db import exception as db_exc -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import inventory as policies -from nova.api.openstack.placement.schemas import inventory as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.db import constants as db_const -from nova.i18n import _ - - -# NOTE(cdent): We keep our own representation of inventory defaults -# and output fields, separate from the versioned object to avoid -# inadvertent API changes when the object defaults are changed. -OUTPUT_INVENTORY_FIELDS = [ - 'total', - 'reserved', - 'min_unit', - 'max_unit', - 'step_size', - 'allocation_ratio', -] -INVENTORY_DEFAULTS = { - 'reserved': 0, - 'min_unit': 1, - 'max_unit': db_const.MAX_INT, - 'step_size': 1, - 'allocation_ratio': 1.0 -} - - -def _extract_inventory(body, schema): - """Extract and validate inventory from JSON body.""" - data = util.extract_json(body, schema) - - inventory_data = copy.copy(INVENTORY_DEFAULTS) - inventory_data.update(data) - - return inventory_data - - -def _extract_inventories(body, schema): - """Extract and validate multiple inventories from JSON body.""" - data = util.extract_json(body, schema) - - inventories = {} - for res_class, raw_inventory in data['inventories'].items(): - inventory_data = copy.copy(INVENTORY_DEFAULTS) - inventory_data.update(raw_inventory) - inventories[res_class] = inventory_data - - data['inventories'] = inventories - return data - - -def make_inventory_object(resource_provider, resource_class, **data): - """Single place to catch malformed Inventories.""" - # TODO(cdent): Some of the validation checks that are done here - # could be done via JSONschema (using, for example, "minimum": - # 0) for non-negative integers. It's not clear if that is - # duplication or decoupling so leaving it as this for now. - try: - inventory = rp_obj.Inventory( - resource_provider=resource_provider, - resource_class=resource_class, **data) - except (ValueError, TypeError) as exc: - raise webob.exc.HTTPBadRequest( - _('Bad inventory %(class)s for resource provider ' - '%(rp_uuid)s: %(error)s') % {'class': resource_class, - 'rp_uuid': resource_provider.uuid, - 'error': exc}) - return inventory - - -def _send_inventories(req, resource_provider, inventories): - """Send a JSON representation of a list of inventories.""" - response = req.response - response.status = 200 - output, last_modified = _serialize_inventories( - inventories, resource_provider.generation) - response.body = encodeutils.to_utf8(jsonutils.dumps(output)) - response.content_type = 'application/json' - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - if want_version.matches((1, 15)): - response.last_modified = last_modified - response.cache_control = 'no-cache' - return response - - -def _send_inventory(req, resource_provider, inventory, status=200): - """Send a JSON representation of one single inventory.""" - response = req.response - response.status = status - response.body = encodeutils.to_utf8(jsonutils.dumps(_serialize_inventory( - inventory, generation=resource_provider.generation))) - response.content_type = 'application/json' - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - if want_version.matches((1, 15)): - modified = util.pick_last_modified(None, inventory) - response.last_modified = modified - response.cache_control = 'no-cache' - return response - - -def _serialize_inventory(inventory, generation=None): - """Turn a single inventory into a dictionary.""" - data = { - field: getattr(inventory, field) - for field in OUTPUT_INVENTORY_FIELDS - } - if generation: - data['resource_provider_generation'] = generation - return data - - -def _serialize_inventories(inventories, generation): - """Turn a list of inventories in a dict by resource class.""" - inventories_by_class = {inventory.resource_class: inventory - for inventory in inventories} - inventories_dict = {} - last_modified = None - for resource_class, inventory in inventories_by_class.items(): - last_modified = util.pick_last_modified(last_modified, inventory) - inventories_dict[resource_class] = _serialize_inventory( - inventory, generation=None) - return ({'resource_provider_generation': generation, - 'inventories': inventories_dict}, last_modified) - - -def _validate_inventory_capacity(version, inventories): - """Validate inventory capacity. - - :param version: request microversion. - :param inventories: Inventory or InventoryList to validate capacities of. - :raises: exception.InvalidInventoryCapacityReservedCanBeTotal if request - microversion is 1.26 or higher and any inventory has capacity < 0. - :raises: exception.InvalidInventoryCapacity if request - microversion is lower than 1.26 and any inventory has capacity <= 0. - """ - if not version.matches((1, 26)): - op = operator.le - exc_class = exception.InvalidInventoryCapacity - else: - op = operator.lt - exc_class = exception.InvalidInventoryCapacityReservedCanBeTotal - if isinstance(inventories, rp_obj.Inventory): - inventories = rp_obj.InventoryList(objects=[inventories]) - for inventory in inventories: - if op(inventory.capacity, 0): - raise exc_class( - resource_class=inventory.resource_class, - resource_provider=inventory.resource_provider.uuid) - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -def create_inventory(req): - """POST to create one inventory. - - On success return a 201 response, a location header pointing - to the newly created inventory and an application/json representation - of the inventory. - """ - context = req.environ['placement.context'] - context.can(policies.CREATE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - data = _extract_inventory(req.body, schema.POST_INVENTORY_SCHEMA) - resource_class = data.pop('resource_class') - - inventory = make_inventory_object(resource_provider, - resource_class, - **data) - - try: - _validate_inventory_capacity( - req.environ[microversion.MICROVERSION_ENVIRON], inventory) - resource_provider.add_inventory(inventory) - except (exception.ConcurrentUpdateDetected, - db_exc.DBDuplicateEntry) as exc: - raise webob.exc.HTTPConflict( - _('Update conflict: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - except (exception.InvalidInventoryCapacity, - exception.NotFound) as exc: - raise webob.exc.HTTPBadRequest( - _('Unable to create inventory for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - - response = req.response - response.location = util.inventory_url( - req.environ, resource_provider, resource_class) - return _send_inventory(req, resource_provider, inventory, - status=201) - - -@wsgi_wrapper.PlacementWsgify -def delete_inventory(req): - """DELETE to destroy a single inventory. - - If the inventory is in use or resource provider generation is out - of sync return a 409. - - On success return a 204 and an empty body. - """ - context = req.environ['placement.context'] - context.can(policies.DELETE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_class = util.wsgi_path_item(req.environ, 'resource_class') - - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - try: - resource_provider.delete_inventory(resource_class) - except (exception.ConcurrentUpdateDetected, - exception.InventoryInUse) as exc: - raise webob.exc.HTTPConflict( - _('Unable to delete inventory of class %(class)s: %(error)s') % - {'class': resource_class, 'error': exc}, - comment=errors.CONCURRENT_UPDATE) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _('No inventory of class %(class)s found for delete: %(error)s') % - {'class': resource_class, 'error': exc}) - - response = req.response - response.status = 204 - response.content_type = None - return response - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def get_inventories(req): - """GET a list of inventories. - - On success return a 200 with an application/json body representing - a collection of inventories. - """ - context = req.environ['placement.context'] - context.can(policies.LIST) - uuid = util.wsgi_path_item(req.environ, 'uuid') - try: - rp = rp_obj.ResourceProvider.get_by_uuid(context, uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("No resource provider with uuid %(uuid)s found : %(error)s") % - {'uuid': uuid, 'error': exc}) - - inv_list = rp_obj.InventoryList.get_all_by_resource_provider(context, rp) - - return _send_inventories(req, rp, inv_list) - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def get_inventory(req): - """GET one inventory. - - On success return a 200 an application/json body representing one - inventory. - """ - context = req.environ['placement.context'] - context.can(policies.SHOW) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_class = util.wsgi_path_item(req.environ, 'resource_class') - try: - rp = rp_obj.ResourceProvider.get_by_uuid(context, uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("No resource provider with uuid %(uuid)s found : %(error)s") % - {'uuid': uuid, 'error': exc}) - - inv_list = rp_obj.InventoryList.get_all_by_resource_provider(context, rp) - inventory = inv_list.find(resource_class) - - if not inventory: - raise webob.exc.HTTPNotFound( - _('No inventory of class %(class)s for %(rp_uuid)s') % - {'class': resource_class, 'rp_uuid': uuid}) - - return _send_inventory(req, rp, inventory) - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -def set_inventories(req): - """PUT to set all inventory for a resource provider. - - Create, update and delete inventory as required to reset all - the inventory. - - If the resource generation is out of sync, return a 409. - If an inventory to be deleted is in use, return a 409. - If any inventory to be created or updated has settings which are - invalid (for example reserved exceeds capacity), return a 400. - - On success return a 200 with an application/json body representing - the inventories. - """ - context = req.environ['placement.context'] - context.can(policies.UPDATE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - data = _extract_inventories(req.body, schema.PUT_INVENTORY_SCHEMA) - if data['resource_provider_generation'] != resource_provider.generation: - raise webob.exc.HTTPConflict( - _('resource provider generation conflict'), - comment=errors.CONCURRENT_UPDATE) - - inv_list = [] - for res_class, inventory_data in data['inventories'].items(): - inventory = make_inventory_object( - resource_provider, res_class, **inventory_data) - inv_list.append(inventory) - inventories = rp_obj.InventoryList(objects=inv_list) - - try: - _validate_inventory_capacity( - req.environ[microversion.MICROVERSION_ENVIRON], inventories) - resource_provider.set_inventory(inventories) - except exception.ResourceClassNotFound as exc: - raise webob.exc.HTTPBadRequest( - _('Unknown resource class in inventory for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - except exception.InventoryWithResourceClassNotFound as exc: - raise webob.exc.HTTPConflict( - _('Race condition detected when setting inventory. No inventory ' - 'record with resource class for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - except (exception.ConcurrentUpdateDetected, - db_exc.DBDuplicateEntry) as exc: - raise webob.exc.HTTPConflict( - _('update conflict: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - except exception.InventoryInUse as exc: - raise webob.exc.HTTPConflict( - _('update conflict: %(error)s') % {'error': exc}, - comment=errors.INVENTORY_INUSE) - except exception.InvalidInventoryCapacity as exc: - raise webob.exc.HTTPBadRequest( - _('Unable to update inventory for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - - return _send_inventories(req, resource_provider, inventories) - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.5', status_code=405) -def delete_inventories(req): - """DELETE all inventory for a resource provider. - - Delete inventory as required to reset all the inventory. - If an inventory to be deleted is in use, return a 409 Conflict. - On success return a 204 No content. - Return 405 Method Not Allowed if the wanted microversion does not match. - """ - context = req.environ['placement.context'] - context.can(policies.DELETE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - inventories = rp_obj.InventoryList(objects=[]) - - try: - resource_provider.set_inventory(inventories) - except exception.ConcurrentUpdateDetected: - raise webob.exc.HTTPConflict( - _('Unable to delete inventory for resource provider ' - '%(rp_uuid)s because the inventory was updated by ' - 'another process. Please retry your request.') - % {'rp_uuid': resource_provider.uuid}, - comment=errors.CONCURRENT_UPDATE) - except exception.InventoryInUse as ex: - # NOTE(mriedem): This message cannot change without impacting the - # nova.scheduler.client.report._RE_INV_IN_USE regex. - raise webob.exc.HTTPConflict(ex.format_message(), - comment=errors.INVENTORY_INUSE) - - response = req.response - response.status = 204 - response.content_type = None - - return response - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -def update_inventory(req): - """PUT to update one inventory. - - If the resource generation is out of sync, return a 409. - If the inventory has settings which are invalid (for example - reserved exceeds capacity), return a 400. - - On success return a 200 with an application/json body representing - the inventory. - """ - context = req.environ['placement.context'] - context.can(policies.UPDATE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - resource_class = util.wsgi_path_item(req.environ, 'resource_class') - - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - data = _extract_inventory(req.body, schema.BASE_INVENTORY_SCHEMA) - if data['resource_provider_generation'] != resource_provider.generation: - raise webob.exc.HTTPConflict( - _('resource provider generation conflict'), - comment=errors.CONCURRENT_UPDATE) - - inventory = make_inventory_object(resource_provider, - resource_class, - **data) - - try: - _validate_inventory_capacity( - req.environ[microversion.MICROVERSION_ENVIRON], inventory) - resource_provider.update_inventory(inventory) - except (exception.ConcurrentUpdateDetected, - db_exc.DBDuplicateEntry) as exc: - raise webob.exc.HTTPConflict( - _('update conflict: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - except exception.InventoryWithResourceClassNotFound as exc: - raise webob.exc.HTTPBadRequest( - _('No inventory record with resource class for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - except exception.InvalidInventoryCapacity as exc: - raise webob.exc.HTTPBadRequest( - _('Unable to update inventory for resource provider ' - '%(rp_uuid)s: %(error)s') % {'rp_uuid': resource_provider.uuid, - 'error': exc}) - - return _send_inventory(req, resource_provider, inventory) diff --git a/nova/api/openstack/placement/handlers/reshaper.py b/nova/api/openstack/placement/handlers/reshaper.py deleted file mode 100644 index 0351ffd286f6..000000000000 --- a/nova/api/openstack/placement/handlers/reshaper.py +++ /dev/null @@ -1,129 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API handler for the reshaper. - -The reshaper provides for atomically migrating resource provider inventories -and associated allocations when some of the inventory moves from one resource -provider to another, such as when a class of inventory moves from a parent -provider to a new child provider. -""" - -import copy - -from oslo_utils import excutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -# TODO(cdent): That we are doing this suggests that there's stuff to be -# extracted from the handler to a shared module. -from nova.api.openstack.placement.handlers import allocation -from nova.api.openstack.placement.handlers import inventory -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import reshaper as policies -from nova.api.openstack.placement.schemas import reshaper as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -# TODO(cdent): placement needs its own version of this -from nova.i18n import _ - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.30') -@util.require_content('application/json') -def reshape(req): - context = req.environ['placement.context'] - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - context.can(policies.RESHAPE) - data = util.extract_json(req.body, schema.POST_RESHAPER_SCHEMA) - inventories = data['inventories'] - allocations = data['allocations'] - # We're going to create several InventoryList, by rp uuid. - inventory_by_rp = {} - - # TODO(cdent): this has overlaps with inventory:set_inventories - # and is a mess of bad names and lack of method extraction. - for rp_uuid, inventory_data in inventories.items(): - try: - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, rp_uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPBadRequest( - _('Resource provider %(rp_uuid)s in inventories not found: ' - '%(error)s') % {'rp_uuid': rp_uuid, 'error': exc}, - comment=errors.RESOURCE_PROVIDER_NOT_FOUND) - - # Do an early generation check. - generation = inventory_data['resource_provider_generation'] - if generation != resource_provider.generation: - raise webob.exc.HTTPConflict( - _('resource provider generation conflict for provider %(rp)s: ' - 'actual: %(actual)s, given: %(given)s') % - {'rp': rp_uuid, - 'actual': resource_provider.generation, - 'given': generation}, - comment=errors.CONCURRENT_UPDATE) - - inv_list = [] - for res_class, raw_inventory in inventory_data['inventories'].items(): - inv_data = copy.copy(inventory.INVENTORY_DEFAULTS) - inv_data.update(raw_inventory) - inv_obj = inventory.make_inventory_object( - resource_provider, res_class, **inv_data) - inv_list.append(inv_obj) - inventory_by_rp[resource_provider] = rp_obj.InventoryList( - objects=inv_list) - - # Make the consumer objects associated with the allocations. - consumers, new_consumers_created = allocation.inspect_consumers( - context, allocations, want_version) - - # Nest exception handling so that any exception results in new consumer - # objects being deleted, then reraise for translating to HTTP exceptions. - try: - try: - # When these allocations are created they get resource provider - # objects which are different instances (usually with the same - # data) from those loaded above when creating inventory objects. - # The reshape method below is responsible for ensuring that the - # resource providers and their generations do not conflict. - allocation_objects = allocation.create_allocation_list( - context, allocations, consumers) - - rp_obj.reshape(context, inventory_by_rp, allocation_objects) - except Exception: - with excutils.save_and_reraise_exception(): - allocation.delete_consumers(new_consumers_created) - # Generation conflict is a (rare) possibility in a few different - # places in reshape(). - except exception.ConcurrentUpdateDetected as exc: - raise webob.exc.HTTPConflict( - _('update conflict: %(error)s') % {'error': exc}, - comment=errors.CONCURRENT_UPDATE) - # A NotFound here means a resource class that does not exist was named - except exception.NotFound as exc: - raise webob.exc.HTTPBadRequest( - _('malformed reshaper data: %(error)s') % {'error': exc}) - # Distinguish inventory in use (has allocations on it)... - except exception.InventoryInUse as exc: - raise webob.exc.HTTPConflict( - _('update conflict: %(error)s') % {'error': exc}, - comment=errors.INVENTORY_INUSE) - # ...from allocations which won't fit for a variety of reasons. - except exception.InvalidInventory as exc: - raise webob.exc.HTTPConflict( - _('Unable to allocate inventory: %(error)s') % {'error': exc}) - - req.response.status = 204 - req.response.content_type = None - return req.response diff --git a/nova/api/openstack/placement/handlers/resource_class.py b/nova/api/openstack/placement/handlers/resource_class.py deleted file mode 100644 index b8b0324a9eb7..000000000000 --- a/nova/api/openstack/placement/handlers/resource_class.py +++ /dev/null @@ -1,241 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API handlers for resource classes.""" - -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -import webob - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import resource_class as policies -from nova.api.openstack.placement.schemas import resource_class as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -def _serialize_links(environ, rc): - url = util.resource_class_url(environ, rc) - links = [{'rel': 'self', 'href': url}] - return links - - -def _serialize_resource_class(environ, rc): - data = { - 'name': rc.name, - 'links': _serialize_links(environ, rc) - } - return data - - -def _serialize_resource_classes(environ, rcs, want_version): - output = [] - last_modified = None - get_last_modified = want_version.matches((1, 15)) - for rc in rcs: - if get_last_modified: - last_modified = util.pick_last_modified(last_modified, rc) - data = _serialize_resource_class(environ, rc) - output.append(data) - last_modified = last_modified or timeutils.utcnow(with_timezone=True) - return ({"resource_classes": output}, last_modified) - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.2') -@util.require_content('application/json') -def create_resource_class(req): - """POST to create a resource class. - - On success return a 201 response with an empty body and a location - header pointing to the newly created resource class. - """ - context = req.environ['placement.context'] - context.can(policies.CREATE) - data = util.extract_json(req.body, schema.POST_RC_SCHEMA_V1_2) - - try: - rc = rp_obj.ResourceClass(context, name=data['name']) - rc.create() - except exception.ResourceClassExists: - raise webob.exc.HTTPConflict( - _('Conflicting resource class already exists: %(name)s') % - {'name': data['name']}) - except exception.MaxDBRetriesExceeded: - raise webob.exc.HTTPConflict( - _('Max retries of DB transaction exceeded attempting ' - 'to create resource class: %(name)s, please ' - 'try again.') % - {'name': data['name']}) - - req.response.location = util.resource_class_url(req.environ, rc) - req.response.status = 201 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.2') -def delete_resource_class(req): - """DELETE to destroy a single resource class. - - On success return a 204 and an empty body. - """ - name = util.wsgi_path_item(req.environ, 'name') - context = req.environ['placement.context'] - context.can(policies.DELETE) - # The containing application will catch a not found here. - rc = rp_obj.ResourceClass.get_by_name(context, name) - try: - rc.destroy() - except exception.ResourceClassCannotDeleteStandard as exc: - raise webob.exc.HTTPBadRequest( - _('Error in delete resource class: %(error)s') % {'error': exc}) - except exception.ResourceClassInUse as exc: - raise webob.exc.HTTPConflict( - _('Error in delete resource class: %(error)s') % {'error': exc}) - req.response.status = 204 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.2') -@util.check_accept('application/json') -def get_resource_class(req): - """Get a single resource class. - - On success return a 200 with an application/json body representing - the resource class. - """ - name = util.wsgi_path_item(req.environ, 'name') - context = req.environ['placement.context'] - context.can(policies.SHOW) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - # The containing application will catch a not found here. - rc = rp_obj.ResourceClass.get_by_name(context, name) - - req.response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_resource_class(req.environ, rc)) - ) - req.response.content_type = 'application/json' - if want_version.matches((1, 15)): - req.response.cache_control = 'no-cache' - # Non-custom resource classes will return None from pick_last_modified, - # so the 'or' causes utcnow to be used. - last_modified = util.pick_last_modified(None, rc) or timeutils.utcnow( - with_timezone=True) - req.response.last_modified = last_modified - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.2') -@util.check_accept('application/json') -def list_resource_classes(req): - """GET a list of resource classes. - - On success return a 200 and an application/json body representing - a collection of resource classes. - """ - context = req.environ['placement.context'] - context.can(policies.LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - rcs = rp_obj.ResourceClassList.get_all(context) - - response = req.response - output, last_modified = _serialize_resource_classes( - req.environ, rcs, want_version) - response.body = encodeutils.to_utf8(jsonutils.dumps(output)) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.last_modified = last_modified - response.cache_control = 'no-cache' - return response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.2', '1.6') -@util.require_content('application/json') -def update_resource_class(req): - """PUT to update a single resource class. - - On success return a 200 response with a representation of the updated - resource class. - """ - name = util.wsgi_path_item(req.environ, 'name') - context = req.environ['placement.context'] - context.can(policies.UPDATE) - - data = util.extract_json(req.body, schema.PUT_RC_SCHEMA_V1_2) - - # The containing application will catch a not found here. - rc = rp_obj.ResourceClass.get_by_name(context, name) - - rc.name = data['name'] - - try: - rc.save() - except exception.ResourceClassExists: - raise webob.exc.HTTPConflict( - _('Resource class already exists: %(name)s') % - {'name': rc.name}) - except exception.ResourceClassCannotUpdateStandard: - raise webob.exc.HTTPBadRequest( - _('Cannot update standard resource class %(rp_name)s') % - {'rp_name': name}) - - req.response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_resource_class(req.environ, rc)) - ) - req.response.status = 200 - req.response.content_type = 'application/json' - return req.response - - -@wsgi_wrapper.PlacementWsgify # noqa -@microversion.version_handler('1.7') -def update_resource_class(req): - """PUT to create or validate the existence of single resource class. - - On a successful create return 201. Return 204 if the class already - exists. If the resource class is not a custom resource class, return - a 400. 409 might be a better choice, but 400 aligns with previous code. - """ - name = util.wsgi_path_item(req.environ, 'name') - context = req.environ['placement.context'] - context.can(policies.UPDATE) - - # Use JSON validation to validation resource class name. - util.extract_json('{"name": "%s"}' % name, schema.PUT_RC_SCHEMA_V1_2) - - status = 204 - try: - rc = rp_obj.ResourceClass.get_by_name(context, name) - except exception.NotFound: - try: - rc = rp_obj.ResourceClass(context, name=name) - rc.create() - status = 201 - # We will not see ResourceClassCannotUpdateStandard because - # that was already caught when validating the {name}. - except exception.ResourceClassExists: - # Someone just now created the class, so stick with 204 - pass - - req.response.status = status - req.response.content_type = None - req.response.location = util.resource_class_url(req.environ, rc) - return req.response diff --git a/nova/api/openstack/placement/handlers/resource_provider.py b/nova/api/openstack/placement/handlers/resource_provider.py deleted file mode 100644 index 8ac38ef6350c..000000000000 --- a/nova/api/openstack/placement/handlers/resource_provider.py +++ /dev/null @@ -1,308 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API handlers for resource providers.""" - -import uuid as uuidlib - -from oslo_db import exception as db_exc -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -from oslo_utils import uuidutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import resource_provider as policies -from nova.api.openstack.placement.schemas import resource_provider as rp_schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -def _serialize_links(environ, resource_provider): - url = util.resource_provider_url(environ, resource_provider) - links = [{'rel': 'self', 'href': url}] - rel_types = ['inventories', 'usages'] - want_version = environ[microversion.MICROVERSION_ENVIRON] - if want_version >= (1, 1): - rel_types.append('aggregates') - if want_version >= (1, 6): - rel_types.append('traits') - if want_version >= (1, 11): - rel_types.append('allocations') - for rel in rel_types: - links.append({'rel': rel, 'href': '%s/%s' % (url, rel)}) - return links - - -def _serialize_provider(environ, resource_provider, want_version): - data = { - 'uuid': resource_provider.uuid, - 'name': resource_provider.name, - 'generation': resource_provider.generation, - 'links': _serialize_links(environ, resource_provider) - } - if want_version.matches((1, 14)): - data['parent_provider_uuid'] = resource_provider.parent_provider_uuid - data['root_provider_uuid'] = resource_provider.root_provider_uuid - return data - - -def _serialize_providers(environ, resource_providers, want_version): - output = [] - last_modified = None - get_last_modified = want_version.matches((1, 15)) - for provider in resource_providers: - if get_last_modified: - last_modified = util.pick_last_modified(last_modified, provider) - provider_data = _serialize_provider(environ, provider, want_version) - output.append(provider_data) - last_modified = last_modified or timeutils.utcnow(with_timezone=True) - return ({"resource_providers": output}, last_modified) - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -def create_resource_provider(req): - """POST to create a resource provider. - - On success return a 201 response with an empty body - (microversions 1.0 - 1.19) or a 200 response with a - payload representing the newly created resource provider - (microversions 1.20 - latest), and a location header - pointing to the resource provider. - """ - context = req.environ['placement.context'] - context.can(policies.CREATE) - schema = rp_schema.POST_RESOURCE_PROVIDER_SCHEMA - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - if want_version.matches((1, 14)): - schema = rp_schema.POST_RP_SCHEMA_V1_14 - data = util.extract_json(req.body, schema) - - try: - if data.get('uuid'): - # Normalize UUID with no proper dashes into dashed one - # with format {8}-{4}-{4}-{4}-{12} - data['uuid'] = str(uuidlib.UUID(data['uuid'])) - else: - data['uuid'] = uuidutils.generate_uuid() - - resource_provider = rp_obj.ResourceProvider(context, **data) - resource_provider.create() - except db_exc.DBDuplicateEntry as exc: - # Whether exc.columns has one or two entries (in the event - # of both fields being duplicates) appears to be database - # dependent, so going with the complete solution here. - duplicate = ', '.join(['%s: %s' % (column, data[column]) - for column in exc.columns]) - raise webob.exc.HTTPConflict( - _('Conflicting resource provider %(duplicate)s already exists.') % - {'duplicate': duplicate}, - comment=errors.DUPLICATE_NAME) - except exception.ObjectActionError as exc: - raise webob.exc.HTTPBadRequest( - _('Unable to create resource provider "%(name)s", %(rp_uuid)s: ' - '%(error)s') % - {'name': data['name'], 'rp_uuid': data['uuid'], 'error': exc}) - - req.response.location = util.resource_provider_url( - req.environ, resource_provider) - if want_version.matches(min_version=(1, 20)): - req.response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_provider(req.environ, resource_provider, want_version))) - req.response.content_type = 'application/json' - modified = util.pick_last_modified(None, resource_provider) - req.response.last_modified = modified - req.response.cache_control = 'no-cache' - else: - req.response.status = 201 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -def delete_resource_provider(req): - """DELETE to destroy a single resource provider. - - On success return a 204 and an empty body. - """ - uuid = util.wsgi_path_item(req.environ, 'uuid') - context = req.environ['placement.context'] - context.can(policies.DELETE) - # The containing application will catch a not found here. - try: - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - resource_provider.destroy() - except exception.ResourceProviderInUse as exc: - raise webob.exc.HTTPConflict( - _('Unable to delete resource provider %(rp_uuid)s: %(error)s') % - {'rp_uuid': uuid, 'error': exc}, - comment=errors.PROVIDER_IN_USE) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("No resource provider with uuid %s found for delete") % uuid) - except exception.CannotDeleteParentResourceProvider as exc: - raise webob.exc.HTTPConflict( - _("Unable to delete parent resource provider %(rp_uuid)s: " - "It has child resource providers.") % {'rp_uuid': uuid}, - comment=errors.PROVIDER_CANNOT_DELETE_PARENT) - req.response.status = 204 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def get_resource_provider(req): - """Get a single resource provider. - - On success return a 200 with an application/json body representing - the resource provider. - """ - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - uuid = util.wsgi_path_item(req.environ, 'uuid') - context = req.environ['placement.context'] - context.can(policies.SHOW) - - # The containing application will catch a not found here. - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - response = req.response - response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_provider(req.environ, resource_provider, want_version))) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - modified = util.pick_last_modified(None, resource_provider) - response.last_modified = modified - response.cache_control = 'no-cache' - return response - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def list_resource_providers(req): - """GET a list of resource providers. - - On success return a 200 and an application/json body representing - a collection of resource providers. - """ - context = req.environ['placement.context'] - context.can(policies.LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - - schema = rp_schema.GET_RPS_SCHEMA_1_0 - if want_version.matches((1, 18)): - schema = rp_schema.GET_RPS_SCHEMA_1_18 - elif want_version.matches((1, 14)): - schema = rp_schema.GET_RPS_SCHEMA_1_14 - elif want_version.matches((1, 4)): - schema = rp_schema.GET_RPS_SCHEMA_1_4 - elif want_version.matches((1, 3)): - schema = rp_schema.GET_RPS_SCHEMA_1_3 - - allow_forbidden = want_version.matches((1, 22)) - - util.validate_query_params(req, schema) - - filters = {} - # special handling of member_of qparam since we allow multiple member_of - # params at microversion 1.24. - if 'member_of' in req.GET: - filters['member_of'] = util.normalize_member_of_qs_params(req) - - qpkeys = ('uuid', 'name', 'in_tree', 'resources', 'required') - for attr in qpkeys: - if attr in req.GET: - value = req.GET[attr] - if attr == 'resources': - value = util.normalize_resources_qs_param(value) - elif attr == 'required': - value = util.normalize_traits_qs_param( - value, allow_forbidden=allow_forbidden) - filters[attr] = value - try: - resource_providers = rp_obj.ResourceProviderList.get_all_by_filters( - context, filters) - except exception.ResourceClassNotFound as exc: - raise webob.exc.HTTPBadRequest( - _('Invalid resource class in resources parameter: %(error)s') % - {'error': exc}) - except exception.TraitNotFound as exc: - raise webob.exc.HTTPBadRequest( - _('Invalid trait(s) in "required" parameter: %(error)s') % - {'error': exc}) - - response = req.response - output, last_modified = _serialize_providers( - req.environ, resource_providers, want_version) - response.body = encodeutils.to_utf8(jsonutils.dumps(output)) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.last_modified = last_modified - response.cache_control = 'no-cache' - return response - - -@wsgi_wrapper.PlacementWsgify -@util.require_content('application/json') -def update_resource_provider(req): - """PUT to update a single resource provider. - - On success return a 200 response with a representation of the updated - resource provider. - """ - uuid = util.wsgi_path_item(req.environ, 'uuid') - context = req.environ['placement.context'] - context.can(policies.UPDATE) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - - # The containing application will catch a not found here. - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - schema = rp_schema.PUT_RESOURCE_PROVIDER_SCHEMA - if want_version.matches((1, 14)): - schema = rp_schema.PUT_RP_SCHEMA_V1_14 - - data = util.extract_json(req.body, schema) - - for field in rp_obj.ResourceProvider.SETTABLE_FIELDS: - if field in data: - setattr(resource_provider, field, data[field]) - - try: - resource_provider.save() - except db_exc.DBDuplicateEntry as exc: - raise webob.exc.HTTPConflict( - _('Conflicting resource provider %(name)s already exists.') % - {'name': data['name']}, - comment=errors.DUPLICATE_NAME) - except exception.ObjectActionError as exc: - raise webob.exc.HTTPBadRequest( - _('Unable to save resource provider %(rp_uuid)s: %(error)s') % - {'rp_uuid': uuid, 'error': exc}) - - response = req.response - response.status = 200 - response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_provider(req.environ, resource_provider, want_version))) - response.content_type = 'application/json' - if want_version.matches((1, 15)): - response.last_modified = resource_provider.updated_at - response.cache_control = 'no-cache' - return response diff --git a/nova/api/openstack/placement/handlers/root.py b/nova/api/openstack/placement/handlers/root.py deleted file mode 100644 index 298dab3816e0..000000000000 --- a/nova/api/openstack/placement/handlers/root.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Handler for the root of the Placement API.""" - -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils - - -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement import wsgi_wrapper - - -@wsgi_wrapper.PlacementWsgify -def home(req): - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - min_version = microversion.min_version_string() - max_version = microversion.max_version_string() - # NOTE(cdent): As sections of the api are added, links can be - # added to this output to align with the guidelines at - # http://specs.openstack.org/openstack/api-wg/guidelines/microversion_specification.html#version-discovery - version_data = { - 'id': 'v%s' % min_version, - 'max_version': max_version, - 'min_version': min_version, - # for now there is only ever one version, so it must be CURRENT - 'status': 'CURRENT', - 'links': [{ - # Point back to this same URL as the root of this version. - # NOTE(cdent): We explicitly want this to be a relative-URL - # representation of "this same URL", otherwise placement needs - # to keep track of proxy addresses and the like, which we have - # avoided thus far, in order to construct full URLs. Placement - # is much easier to scale if we never track that stuff. - 'rel': 'self', - 'href': '', - }], - } - version_json = jsonutils.dumps({'versions': [version_data]}) - req.response.body = encodeutils.to_utf8(version_json) - req.response.content_type = 'application/json' - if want_version.matches((1, 15)): - req.response.cache_control = 'no-cache' - req.response.last_modified = timeutils.utcnow(with_timezone=True) - return req.response diff --git a/nova/api/openstack/placement/handlers/trait.py b/nova/api/openstack/placement/handlers/trait.py deleted file mode 100644 index b76907f1ad9f..000000000000 --- a/nova/api/openstack/placement/handlers/trait.py +++ /dev/null @@ -1,270 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Traits handlers for Placement API.""" - -import jsonschema -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import trait as policies -from nova.api.openstack.placement.schemas import trait as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -def _normalize_traits_qs_param(qs): - try: - op, value = qs.split(':', 1) - except ValueError: - msg = _('Badly formatted name parameter. Expected name query string ' - 'parameter in form: ' - '?name=[in|startswith]:[name1,name2|prefix]. Got: "%s"') - msg = msg % qs - raise webob.exc.HTTPBadRequest(msg) - - filters = {} - if op == 'in': - filters['name_in'] = value.split(',') - elif op == 'startswith': - filters['prefix'] = value - - return filters - - -def _serialize_traits(traits, want_version): - last_modified = None - get_last_modified = want_version.matches((1, 15)) - trait_names = [] - for trait in traits: - if get_last_modified: - last_modified = util.pick_last_modified(last_modified, trait) - trait_names.append(trait.name) - - # If there were no traits, set last_modified to now - last_modified = last_modified or timeutils.utcnow(with_timezone=True) - - return {'traits': trait_names}, last_modified - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -def put_trait(req): - context = req.environ['placement.context'] - context.can(policies.TRAITS_UPDATE) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - name = util.wsgi_path_item(req.environ, 'name') - - try: - jsonschema.validate(name, schema.CUSTOM_TRAIT) - except jsonschema.ValidationError: - raise webob.exc.HTTPBadRequest( - _('The trait is invalid. A valid trait must be no longer than ' - '255 characters, start with the prefix "CUSTOM_" and use ' - 'following characters: "A"-"Z", "0"-"9" and "_"')) - - trait = rp_obj.Trait(context) - trait.name = name - - try: - trait.create() - req.response.status = 201 - except exception.TraitExists: - # Get the trait that already exists to get last-modified time. - if want_version.matches((1, 15)): - trait = rp_obj.Trait.get_by_name(context, name) - req.response.status = 204 - - req.response.content_type = None - req.response.location = util.trait_url(req.environ, trait) - if want_version.matches((1, 15)): - req.response.last_modified = trait.created_at - req.response.cache_control = 'no-cache' - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -def get_trait(req): - context = req.environ['placement.context'] - context.can(policies.TRAITS_SHOW) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - name = util.wsgi_path_item(req.environ, 'name') - - try: - trait = rp_obj.Trait.get_by_name(context, name) - except exception.TraitNotFound as ex: - raise webob.exc.HTTPNotFound(ex.format_message()) - - req.response.status = 204 - req.response.content_type = None - if want_version.matches((1, 15)): - req.response.last_modified = trait.created_at - req.response.cache_control = 'no-cache' - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -def delete_trait(req): - context = req.environ['placement.context'] - context.can(policies.TRAITS_DELETE) - name = util.wsgi_path_item(req.environ, 'name') - - try: - trait = rp_obj.Trait.get_by_name(context, name) - trait.destroy() - except exception.TraitNotFound as ex: - raise webob.exc.HTTPNotFound(ex.format_message()) - except exception.TraitCannotDeleteStandard as ex: - raise webob.exc.HTTPBadRequest(ex.format_message()) - except exception.TraitInUse as ex: - raise webob.exc.HTTPConflict(ex.format_message()) - - req.response.status = 204 - req.response.content_type = None - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -@util.check_accept('application/json') -def list_traits(req): - context = req.environ['placement.context'] - context.can(policies.TRAITS_LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - filters = {} - - util.validate_query_params(req, schema.LIST_TRAIT_SCHEMA) - - if 'name' in req.GET: - filters = _normalize_traits_qs_param(req.GET['name']) - if 'associated' in req.GET: - if req.GET['associated'].lower() not in ['true', 'false']: - raise webob.exc.HTTPBadRequest( - _('The query parameter "associated" only accepts ' - '"true" or "false"')) - filters['associated'] = ( - True if req.GET['associated'].lower() == 'true' else False) - - traits = rp_obj.TraitList.get_all(context, filters) - req.response.status = 200 - output, last_modified = _serialize_traits(traits, want_version) - if want_version.matches((1, 15)): - req.response.last_modified = last_modified - req.response.cache_control = 'no-cache' - req.response.body = encodeutils.to_utf8(jsonutils.dumps(output)) - req.response.content_type = 'application/json' - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -@util.check_accept('application/json') -def list_traits_for_resource_provider(req): - context = req.environ['placement.context'] - context.can(policies.RP_TRAIT_LIST) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - uuid = util.wsgi_path_item(req.environ, 'uuid') - - # Resource provider object is needed for two things: If it is - # NotFound we'll get a 404 here, which needs to happen because - # get_all_by_resource_provider can return an empty list. - # It is also needed for the generation, used in the outgoing - # representation. - try: - rp = rp_obj.ResourceProvider.get_by_uuid(context, uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("No resource provider with uuid %(uuid)s found: %(error)s") % - {'uuid': uuid, 'error': exc}) - - traits = rp_obj.TraitList.get_all_by_resource_provider(context, rp) - response_body, last_modified = _serialize_traits(traits, want_version) - response_body["resource_provider_generation"] = rp.generation - - if want_version.matches((1, 15)): - req.response.last_modified = last_modified - req.response.cache_control = 'no-cache' - - req.response.status = 200 - req.response.body = encodeutils.to_utf8(jsonutils.dumps(response_body)) - req.response.content_type = 'application/json' - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -@util.require_content('application/json') -def update_traits_for_resource_provider(req): - context = req.environ['placement.context'] - context.can(policies.RP_TRAIT_UPDATE) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - uuid = util.wsgi_path_item(req.environ, 'uuid') - data = util.extract_json(req.body, schema.SET_TRAITS_FOR_RP_SCHEMA) - rp_gen = data['resource_provider_generation'] - traits = data['traits'] - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - - if resource_provider.generation != rp_gen: - raise webob.exc.HTTPConflict( - _("Resource provider's generation already changed. Please update " - "the generation and try again."), - json_formatter=util.json_error_formatter, - comment=errors.CONCURRENT_UPDATE) - - trait_objs = rp_obj.TraitList.get_all( - context, filters={'name_in': traits}) - traits_name = set([obj.name for obj in trait_objs]) - non_existed_trait = set(traits) - set(traits_name) - if non_existed_trait: - raise webob.exc.HTTPBadRequest( - _("No such trait %s") % ', '.join(non_existed_trait)) - - resource_provider.set_traits(trait_objs) - - response_body, last_modified = _serialize_traits(trait_objs, want_version) - response_body[ - 'resource_provider_generation'] = resource_provider.generation - if want_version.matches((1, 15)): - req.response.last_modified = last_modified - req.response.cache_control = 'no-cache' - req.response.status = 200 - req.response.body = encodeutils.to_utf8(jsonutils.dumps(response_body)) - req.response.content_type = 'application/json' - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.6') -def delete_traits_for_resource_provider(req): - context = req.environ['placement.context'] - context.can(policies.RP_TRAIT_DELETE) - uuid = util.wsgi_path_item(req.environ, 'uuid') - - resource_provider = rp_obj.ResourceProvider.get_by_uuid(context, uuid) - try: - resource_provider.set_traits(rp_obj.TraitList(objects=[])) - except exception.ConcurrentUpdateDetected as e: - raise webob.exc.HTTPConflict(e.format_message(), - comment=errors.CONCURRENT_UPDATE) - - req.response.status = 204 - req.response.content_type = None - return req.response diff --git a/nova/api/openstack/placement/handlers/usage.py b/nova/api/openstack/placement/handlers/usage.py deleted file mode 100644 index 85213302d46a..000000000000 --- a/nova/api/openstack/placement/handlers/usage.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API handlers for usage information.""" - -from oslo_serialization import jsonutils -from oslo_utils import encodeutils -from oslo_utils import timeutils -import webob - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import microversion -from nova.api.openstack.placement.objects import resource_provider as rp_obj -from nova.api.openstack.placement.policies import usage as policies -from nova.api.openstack.placement.schemas import usage as schema -from nova.api.openstack.placement import util -from nova.api.openstack.placement import wsgi_wrapper -from nova.i18n import _ - - -def _serialize_usages(resource_provider, usage): - usage_dict = {resource.resource_class: resource.usage - for resource in usage} - return {'resource_provider_generation': resource_provider.generation, - 'usages': usage_dict} - - -@wsgi_wrapper.PlacementWsgify -@util.check_accept('application/json') -def list_usages(req): - """GET a dictionary of resource provider usage by resource class. - - If the resource provider does not exist return a 404. - - On success return a 200 with an application/json representation of - the usage dictionary. - """ - context = req.environ['placement.context'] - context.can(policies.PROVIDER_USAGES) - uuid = util.wsgi_path_item(req.environ, 'uuid') - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - - # Resource provider object needed for two things: If it is - # NotFound we'll get a 404 here, which needs to happen because - # get_all_by_resource_provider_uuid can return an empty list. - # It is also needed for the generation, used in the outgoing - # representation. - try: - resource_provider = rp_obj.ResourceProvider.get_by_uuid( - context, uuid) - except exception.NotFound as exc: - raise webob.exc.HTTPNotFound( - _("No resource provider with uuid %(uuid)s found: %(error)s") % - {'uuid': uuid, 'error': exc}) - - usage = rp_obj.UsageList.get_all_by_resource_provider_uuid( - context, uuid) - - response = req.response - response.body = encodeutils.to_utf8(jsonutils.dumps( - _serialize_usages(resource_provider, usage))) - req.response.content_type = 'application/json' - if want_version.matches((1, 15)): - req.response.cache_control = 'no-cache' - # While it would be possible to generate a last-modified time - # based on the collection of allocations that result in a usage - # value (with some spelunking in the SQL) that doesn't align with - # the question that is being asked in a request for usages: What - # is the usage, now? So the last-modified time is set to utcnow. - req.response.last_modified = timeutils.utcnow(with_timezone=True) - return req.response - - -@wsgi_wrapper.PlacementWsgify -@microversion.version_handler('1.9') -@util.check_accept('application/json') -def get_total_usages(req): - """GET the sum of usages for a project or a project/user. - - On success return a 200 and an application/json body representing the - sum/total of usages. - Return 404 Not Found if the wanted microversion does not match. - """ - context = req.environ['placement.context'] - # TODO(mriedem): When we support non-admins to use GET /usages we - # should pass the project_id (and user_id?) from the query parameters - # into context.can() for the target. - context.can(policies.TOTAL_USAGES) - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - - util.validate_query_params(req, schema.GET_USAGES_SCHEMA_1_9) - - project_id = req.GET.get('project_id') - user_id = req.GET.get('user_id') - - usages = rp_obj.UsageList.get_all_by_project_user(context, project_id, - user_id=user_id) - - response = req.response - usages_dict = {'usages': {resource.resource_class: resource.usage - for resource in usages}} - response.body = encodeutils.to_utf8(jsonutils.dumps(usages_dict)) - req.response.content_type = 'application/json' - if want_version.matches((1, 15)): - req.response.cache_control = 'no-cache' - # While it would be possible to generate a last-modified time - # based on the collection of allocations that result in a usage - # value (with some spelunking in the SQL) that doesn't align with - # the question that is being asked in a request for usages: What - # is the usage, now? So the last-modified time is set to utcnow. - req.response.last_modified = timeutils.utcnow(with_timezone=True) - return req.response diff --git a/nova/api/openstack/placement/lib.py b/nova/api/openstack/placement/lib.py deleted file mode 100644 index 0518027c69e3..000000000000 --- a/nova/api/openstack/placement/lib.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Symbols intended to be imported by both placement code and placement API -consumers. When placement is separated out, this module should be part of a -common library that both placement and its consumers can require.""" - - -class RequestGroup(object): - def __init__(self, use_same_provider=True, resources=None, - required_traits=None, forbidden_traits=None, member_of=None): - """Create a grouping of resource and trait requests. - - :param use_same_provider: - If True, (the default) this RequestGroup represents requests for - resources and traits which must be satisfied by a single resource - provider. If False, represents a request for resources and traits - in any resource provider in the same tree, or a sharing provider. - :param resources: A dict of { resource_class: amount, ... } - :param required_traits: A set of { trait_name, ... } - :param forbidden_traits: A set of { trait_name, ... } - :param member_of: A list of [ [aggregate_UUID], - [aggregate_UUID, aggregate_UUID] ... ] - """ - self.use_same_provider = use_same_provider - self.resources = resources or {} - self.required_traits = required_traits or set() - self.forbidden_traits = forbidden_traits or set() - self.member_of = member_of or [] - - def __str__(self): - ret = 'RequestGroup(use_same_provider=%s' % str(self.use_same_provider) - ret += ', resources={%s}' % ', '.join( - '%s:%d' % (rc, amount) - for rc, amount in sorted(list(self.resources.items()))) - ret += ', traits=[%s]' % ', '.join( - sorted(self.required_traits) + - ['!%s' % ft for ft in sorted(self.forbidden_traits)]) - ret += ', aggregates=[%s]' % ', '.join( - sorted('[%s]' % ', '.join(agglist) - for agglist in sorted(self.member_of))) - ret += ')' - return ret diff --git a/nova/api/openstack/placement/microversion.py b/nova/api/openstack/placement/microversion.py deleted file mode 100644 index c832a1a4ed14..000000000000 --- a/nova/api/openstack/placement/microversion.py +++ /dev/null @@ -1,172 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Microversion handling.""" - -# NOTE(cdent): This code is taken from enamel: -# https://github.com/jaypipes/enamel and was the original source of -# the code now used in microversion_parse library. - -import collections -import inspect - -import microversion_parse -import webob - - -SERVICE_TYPE = 'placement' -MICROVERSION_ENVIRON = '%s.microversion' % SERVICE_TYPE -VERSIONED_METHODS = collections.defaultdict(list) - -# The Canonical Version List -VERSIONS = [ - '1.0', - '1.1', # initial support for aggregate.get_aggregates and set_aggregates - '1.2', # Adds /resource_classes resource endpoint - '1.3', # Adds 'member_of' query parameter to get resource providers - # that are members of any of the listed aggregates - '1.4', # Adds resources query string parameter in GET /resource_providers - '1.5', # Adds DELETE /resource_providers/{uuid}/inventories - '1.6', # Adds /traits and /resource_providers{uuid}/traits resource - # endpoints - '1.7', # PUT /resource_classes/{name} is bodiless create or update - '1.8', # Adds 'project_id' and 'user_id' required request parameters to - # PUT /allocations - '1.9', # Adds GET /usages - '1.10', # Adds GET /allocation_candidates resource endpoint - '1.11', # Adds 'allocations' link to the GET /resource_providers response - '1.12', # Add project_id and user_id to GET /allocations/{consumer_uuid} - # and PUT to /allocations/{consumer_uuid} in the same dict form - # as GET. The 'allocation_requests' format in GET - # /allocation_candidates is updated to be the same as well. - '1.13', # Adds POST /allocations to set allocations for multiple consumers - '1.14', # Adds parent and root provider UUID on resource provider - # representation and 'in_tree' filter on GET /resource_providers - '1.15', # Include last-modified and cache-control headers - '1.16', # Add 'limit' query parameter to GET /allocation_candidates - '1.17', # Add 'required' query parameter to GET /allocation_candidates and - # return traits in the provider summary. - '1.18', # Support ?required= queryparam on GET /resource_providers - '1.19', # Include generation and conflict detection in provider aggregates - # APIs - '1.20', # Return 200 with provider payload from POST /resource_providers - '1.21', # Support ?member_of=in: queryparam on - # GET /allocation_candidates - '1.22', # Support forbidden traits in the required parameter of - # GET /resource_providers and GET /allocation_candidates - '1.23', # Add support for error codes in error response JSON - '1.24', # Support multiple ?member_of= queryparams on - # GET /resource_providers - '1.25', # Adds support for granular resource requests via numbered - # querystring groups in GET /allocation_candidates - '1.26', # Add ability to specify inventory with reserved value equal to - # total. - '1.27', # Include all resource class inventories in `provider_summaries` - # field in response of `GET /allocation_candidates` API even if - # the resource class is not in the requested resources. - '1.28', # Add support for consumer generation - '1.29', # Support nested providers in GET /allocation_candidates API. - '1.30', # Add POST /reshaper for atomically migrating resource provider - # inventories and allocations. -] - - -def max_version_string(): - return VERSIONS[-1] - - -def min_version_string(): - return VERSIONS[0] - - -# From twisted -# https://github.com/twisted/twisted/blob/trunk/twisted/python/deprecate.py -def _fully_qualified_name(obj): - """Return the fully qualified name of a module, class, method or function. - - Classes and functions need to be module level ones to be correctly - qualified. - """ - try: - name = obj.__qualname__ - except AttributeError: - name = obj.__name__ - - if inspect.isclass(obj) or inspect.isfunction(obj): - moduleName = obj.__module__ - return "%s.%s" % (moduleName, name) - elif inspect.ismethod(obj): - try: - cls = obj.im_class - except AttributeError: - # Python 3 eliminates im_class, substitutes __module__ and - # __qualname__ to provide similar information. - return "%s.%s" % (obj.__module__, obj.__qualname__) - else: - className = _fully_qualified_name(cls) - return "%s.%s" % (className, name) - return name - - -def _find_method(f, version, status_code): - """Look in VERSIONED_METHODS for method with right name matching version. - - If no match is found a HTTPError corresponding to status_code will - be returned. - """ - qualified_name = _fully_qualified_name(f) - # A KeyError shouldn't be possible here, but let's be robust - # just in case. - method_list = VERSIONED_METHODS.get(qualified_name, []) - for min_version, max_version, func in method_list: - if min_version <= version <= max_version: - return func - - raise webob.exc.status_map[status_code] - - -def version_handler(min_ver, max_ver=None, status_code=404): - """Decorator for versioning API methods. - - Add as a decorator to a placement API handler to constrain - the microversions at which it will run. Add after the - ``wsgify`` decorator. - - This does not check for version intersections. That's the - domain of tests. - - :param min_ver: A string of two numerals, X.Y indicating the - minimum version allowed for the decorated method. - :param max_ver: A string of two numerals, X.Y, indicating the - maximum version allowed for the decorated method. - :param status_code: A status code to indicate error, 404 by default - """ - def decorator(f): - min_version = microversion_parse.parse_version_string(min_ver) - if max_ver: - max_version = microversion_parse.parse_version_string(max_ver) - else: - max_version = microversion_parse.parse_version_string( - max_version_string()) - qualified_name = _fully_qualified_name(f) - VERSIONED_METHODS[qualified_name].append( - (min_version, max_version, f)) - - def decorated_func(req, *args, **kwargs): - version = req.environ[MICROVERSION_ENVIRON] - return _find_method(f, version, status_code)(req, *args, **kwargs) - - # Sort highest min version to beginning of list. - VERSIONED_METHODS[qualified_name].sort(key=lambda x: x[0], - reverse=True) - return decorated_func - return decorator diff --git a/nova/api/openstack/placement/objects/__init__.py b/nova/api/openstack/placement/objects/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/api/openstack/placement/objects/consumer.py b/nova/api/openstack/placement/objects/consumer.py deleted file mode 100644 index 9d88a83adc4d..000000000000 --- a/nova/api/openstack/placement/objects/consumer.py +++ /dev/null @@ -1,257 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_db import exception as db_exc -from oslo_versionedobjects import base -from oslo_versionedobjects import fields -import sqlalchemy as sa - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import exception -from nova.api.openstack.placement.objects import project as project_obj -from nova.api.openstack.placement.objects import user as user_obj -from nova.db.sqlalchemy import api_models as models - -CONSUMER_TBL = models.Consumer.__table__ -_ALLOC_TBL = models.Allocation.__table__ - - -@db_api.placement_context_manager.writer -def create_incomplete_consumers(ctx, batch_size): - """Finds all the consumer records that are missing for allocations and - creates consumer records for them, using the "incomplete consumer" project - and user CONF options. - - Returns a tuple containing two identical elements with the number of - consumer records created, since this is the expected return format for data - migration routines. - """ - # Create a record in the projects table for our incomplete project - incomplete_proj_id = project_obj.ensure_incomplete_project(ctx) - - # Create a record in the users table for our incomplete user - incomplete_user_id = user_obj.ensure_incomplete_user(ctx) - - # Create a consumer table record for all consumers where - # allocations.consumer_id doesn't exist in the consumers table. Use the - # incomplete consumer project and user ID. - alloc_to_consumer = sa.outerjoin( - _ALLOC_TBL, CONSUMER_TBL, - _ALLOC_TBL.c.consumer_id == CONSUMER_TBL.c.uuid) - cols = [ - _ALLOC_TBL.c.consumer_id, - incomplete_proj_id, - incomplete_user_id, - ] - sel = sa.select(cols) - sel = sel.select_from(alloc_to_consumer) - sel = sel.where(CONSUMER_TBL.c.id.is_(None)) - # NOTE(mnaser): It is possible to have multiple consumers having many - # allocations to the same resource provider, which would - # make the INSERT FROM SELECT fail due to duplicates. - sel = sel.group_by(_ALLOC_TBL.c.consumer_id) - sel = sel.limit(batch_size) - target_cols = ['uuid', 'project_id', 'user_id'] - ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel) - res = ctx.session.execute(ins_stmt) - return res.rowcount, res.rowcount - - -@db_api.placement_context_manager.writer -def delete_consumers_if_no_allocations(ctx, consumer_uuids): - """Looks to see if any of the supplied consumers has any allocations and if - not, deletes the consumer record entirely. - - :param ctx: `nova.api.openstack.placement.context.RequestContext` that - contains an oslo_db Session - :param consumer_uuids: UUIDs of the consumers to check and maybe delete - """ - # Delete consumers that are not referenced in the allocations table - cons_to_allocs_join = sa.outerjoin( - CONSUMER_TBL, _ALLOC_TBL, - CONSUMER_TBL.c.uuid == _ALLOC_TBL.c.consumer_id) - subq = sa.select([CONSUMER_TBL.c.uuid]).select_from(cons_to_allocs_join) - subq = subq.where(sa.and_( - _ALLOC_TBL.c.consumer_id.is_(None), - CONSUMER_TBL.c.uuid.in_(consumer_uuids))) - no_alloc_consumers = [r[0] for r in ctx.session.execute(subq).fetchall()] - del_stmt = CONSUMER_TBL.delete() - del_stmt = del_stmt.where(CONSUMER_TBL.c.uuid.in_(no_alloc_consumers)) - ctx.session.execute(del_stmt) - - -@db_api.placement_context_manager.reader -def _get_consumer_by_uuid(ctx, uuid): - # The SQL for this looks like the following: - # SELECT - # c.id, c.uuid, - # p.id AS project_id, p.external_id AS project_external_id, - # u.id AS user_id, u.external_id AS user_external_id, - # c.updated_at, c.created_at - # FROM consumers c - # INNER JOIN projects p - # ON c.project_id = p.id - # INNER JOIN users u - # ON c.user_id = u.id - # WHERE c.uuid = $uuid - consumers = sa.alias(CONSUMER_TBL, name="c") - projects = sa.alias(project_obj.PROJECT_TBL, name="p") - users = sa.alias(user_obj.USER_TBL, name="u") - cols = [ - consumers.c.id, - consumers.c.uuid, - projects.c.id.label("project_id"), - projects.c.external_id.label("project_external_id"), - users.c.id.label("user_id"), - users.c.external_id.label("user_external_id"), - consumers.c.generation, - consumers.c.updated_at, - consumers.c.created_at - ] - c_to_p_join = sa.join( - consumers, projects, consumers.c.project_id == projects.c.id) - c_to_u_join = sa.join( - c_to_p_join, users, consumers.c.user_id == users.c.id) - sel = sa.select(cols).select_from(c_to_u_join) - sel = sel.where(consumers.c.uuid == uuid) - res = ctx.session.execute(sel).fetchone() - if not res: - raise exception.ConsumerNotFound(uuid=uuid) - - return dict(res) - - -@db_api.placement_context_manager.writer -def _increment_consumer_generation(ctx, consumer): - """Increments the supplied consumer's generation value, supplying the - consumer object which contains the currently-known generation. Returns the - newly-incremented generation. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param consumer: `Consumer` whose generation should be updated. - :returns: The newly-incremented generation. - :raises nova.exception.ConcurrentUpdateDetected: if another thread updated - the same consumer's view of its allocations in between the time - when this object was originally read and the call which modified - the consumer's state (e.g. replacing allocations for a consumer) - """ - consumer_gen = consumer.generation - new_generation = consumer_gen + 1 - upd_stmt = CONSUMER_TBL.update().where(sa.and_( - CONSUMER_TBL.c.id == consumer.id, - CONSUMER_TBL.c.generation == consumer_gen)).values( - generation=new_generation) - - res = ctx.session.execute(upd_stmt) - if res.rowcount != 1: - raise exception.ConcurrentUpdateDetected - return new_generation - - -@db_api.placement_context_manager.writer -def _delete_consumer(ctx, consumer): - """Deletes the supplied consumer. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param consumer: `Consumer` whose generation should be updated. - """ - del_stmt = CONSUMER_TBL.delete().where(CONSUMER_TBL.c.id == consumer.id) - ctx.session.execute(del_stmt) - - -@base.VersionedObjectRegistry.register_if(False) -class Consumer(base.VersionedObject, base.TimestampedObject): - - fields = { - 'id': fields.IntegerField(read_only=True), - 'uuid': fields.UUIDField(nullable=False), - 'project': fields.ObjectField('Project', nullable=False), - 'user': fields.ObjectField('User', nullable=False), - 'generation': fields.IntegerField(nullable=False), - } - - @staticmethod - def _from_db_object(ctx, target, source): - target.id = source['id'] - target.uuid = source['uuid'] - target.generation = source['generation'] - target.created_at = source['created_at'] - target.updated_at = source['updated_at'] - - target.project = project_obj.Project( - ctx, id=source['project_id'], - external_id=source['project_external_id']) - target.user = user_obj.User( - ctx, id=source['user_id'], - external_id=source['user_external_id']) - - target._context = ctx - target.obj_reset_changes() - return target - - @classmethod - def get_by_uuid(cls, ctx, uuid): - res = _get_consumer_by_uuid(ctx, uuid) - return cls._from_db_object(ctx, cls(ctx), res) - - def create(self): - @db_api.placement_context_manager.writer - def _create_in_db(ctx): - db_obj = models.Consumer( - uuid=self.uuid, project_id=self.project.id, - user_id=self.user.id) - try: - db_obj.save(ctx.session) - # NOTE(jaypipes): We don't do the normal _from_db_object() - # thing here because models.Consumer doesn't have a - # project_external_id or user_external_id attribute. - self.id = db_obj.id - self.generation = db_obj.generation - except db_exc.DBDuplicateEntry: - raise exception.ConsumerExists(uuid=self.uuid) - _create_in_db(self._context) - self.obj_reset_changes() - - def update(self): - """Used to update the consumer's project and user information without - incrementing the consumer's generation. - """ - @db_api.placement_context_manager.writer - def _update_in_db(ctx): - upd_stmt = CONSUMER_TBL.update().values( - project_id=self.project.id, user_id=self.user.id) - # NOTE(jaypipes): We add the generation check to the WHERE clause - # above just for safety. We don't need to check that the statement - # actually updated a single row. If it did not, then the - # consumer.increment_generation() call that happens in - # AllocationList.replace_all() will end up raising - # ConcurrentUpdateDetected anyway - upd_stmt = upd_stmt.where(sa.and_( - CONSUMER_TBL.c.id == self.id, - CONSUMER_TBL.c.generation == self.generation)) - ctx.session.execute(upd_stmt) - _update_in_db(self._context) - self.obj_reset_changes() - - def increment_generation(self): - """Increments the consumer's generation. - - :raises nova.exception.ConcurrentUpdateDetected: if another thread - updated the same consumer's view of its allocations in between the - time when this object was originally read and the call which - modified the consumer's state (e.g. replacing allocations for a - consumer) - """ - self.generation = _increment_consumer_generation(self._context, self) - - def delete(self): - _delete_consumer(self._context, self) diff --git a/nova/api/openstack/placement/objects/project.py b/nova/api/openstack/placement/objects/project.py deleted file mode 100644 index a6742da2fdce..000000000000 --- a/nova/api/openstack/placement/objects/project.py +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_db import exception as db_exc -from oslo_versionedobjects import base -from oslo_versionedobjects import fields -import sqlalchemy as sa - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import exception -from nova.db.sqlalchemy import api_models as models - -CONF = cfg.CONF -PROJECT_TBL = models.Project.__table__ - - -@db_api.placement_context_manager.writer -def ensure_incomplete_project(ctx): - """Ensures that a project record is created for the "incomplete consumer - project". Returns the internal ID of that record. - """ - incomplete_id = CONF.placement.incomplete_consumer_project_id - sel = sa.select([PROJECT_TBL.c.id]).where( - PROJECT_TBL.c.external_id == incomplete_id) - res = ctx.session.execute(sel).fetchone() - if res: - return res[0] - ins = PROJECT_TBL.insert().values(external_id=incomplete_id) - res = ctx.session.execute(ins) - return res.inserted_primary_key[0] - - -@db_api.placement_context_manager.reader -def _get_project_by_external_id(ctx, external_id): - projects = sa.alias(PROJECT_TBL, name="p") - cols = [ - projects.c.id, - projects.c.external_id, - projects.c.updated_at, - projects.c.created_at - ] - sel = sa.select(cols) - sel = sel.where(projects.c.external_id == external_id) - res = ctx.session.execute(sel).fetchone() - if not res: - raise exception.ProjectNotFound(external_id=external_id) - - return dict(res) - - -@base.VersionedObjectRegistry.register_if(False) -class Project(base.VersionedObject): - - fields = { - 'id': fields.IntegerField(read_only=True), - 'external_id': fields.StringField(nullable=False), - } - - @staticmethod - def _from_db_object(ctx, target, source): - for field in target.fields: - setattr(target, field, source[field]) - - target._context = ctx - target.obj_reset_changes() - return target - - @classmethod - def get_by_external_id(cls, ctx, external_id): - res = _get_project_by_external_id(ctx, external_id) - return cls._from_db_object(ctx, cls(ctx), res) - - def create(self): - @db_api.placement_context_manager.writer - def _create_in_db(ctx): - db_obj = models.Project(external_id=self.external_id) - try: - db_obj.save(ctx.session) - except db_exc.DBDuplicateEntry: - raise exception.ProjectExists(external_id=self.external_id) - self._from_db_object(ctx, self, db_obj) - _create_in_db(self._context) diff --git a/nova/api/openstack/placement/objects/resource_provider.py b/nova/api/openstack/placement/objects/resource_provider.py deleted file mode 100644 index 87e75ec65305..000000000000 --- a/nova/api/openstack/placement/objects/resource_provider.py +++ /dev/null @@ -1,4282 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -import copy -import itertools -import random - -# NOTE(cdent): The resource provider objects are designed to never be -# used over RPC. Remote manipulation is done with the placement HTTP -# API. The 'remotable' decorators should not be used, the objects should -# not be registered and there is no need to express VERSIONs nor handle -# obj_make_compatible. - -import os_traits -from oslo_concurrency import lockutils -from oslo_config import cfg -from oslo_db import api as oslo_db_api -from oslo_db import exception as db_exc -from oslo_log import log as logging -from oslo_utils import encodeutils -from oslo_versionedobjects import base -from oslo_versionedobjects import fields -import six -import sqlalchemy as sa -from sqlalchemy import exc as sqla_exc -from sqlalchemy import func -from sqlalchemy import sql -from sqlalchemy.sql import null - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import exception -from nova.api.openstack.placement.objects import consumer as consumer_obj -from nova.api.openstack.placement.objects import project as project_obj -from nova.api.openstack.placement.objects import user as user_obj -from nova.api.openstack.placement import resource_class_cache as rc_cache -from nova.db.sqlalchemy import api_models as models -from nova.i18n import _ -from nova import rc_fields - -_TRAIT_TBL = models.Trait.__table__ -_ALLOC_TBL = models.Allocation.__table__ -_INV_TBL = models.Inventory.__table__ -_RP_TBL = models.ResourceProvider.__table__ -# Not used in this file but used in tests. -_RC_TBL = models.ResourceClass.__table__ -_AGG_TBL = models.PlacementAggregate.__table__ -_RP_AGG_TBL = models.ResourceProviderAggregate.__table__ -_RP_TRAIT_TBL = models.ResourceProviderTrait.__table__ -_PROJECT_TBL = models.Project.__table__ -_USER_TBL = models.User.__table__ -_CONSUMER_TBL = models.Consumer.__table__ -_RC_CACHE = None -_TRAIT_LOCK = 'trait_sync' -_TRAITS_SYNCED = False - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -@db_api.placement_context_manager.reader -def ensure_rc_cache(ctx): - """Ensures that a singleton resource class cache has been created in the - module's scope. - - :param ctx: `nova.context.RequestContext` that may be used to grab a DB - connection. - """ - global _RC_CACHE - if _RC_CACHE is not None: - return - _RC_CACHE = rc_cache.ResourceClassCache(ctx) - - -@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) -# Bug #1760322: If the caller raises an exception, we don't want the trait -# sync rolled back; so use an .independent transaction -@db_api.placement_context_manager.writer.independent -def _trait_sync(ctx): - """Sync the os_traits symbols to the database. - - Reads all symbols from the os_traits library, checks if any of them do - not exist in the database and bulk-inserts those that are not. This is - done once per process using this code if either Trait.get_by_name or - TraitList.get_all is called. - - :param ctx: `nova.context.RequestContext` that may be used to grab a DB - connection. - """ - # Create a set of all traits in the os_traits library. - std_traits = set(os_traits.get_traits()) - sel = sa.select([_TRAIT_TBL.c.name]) - res = ctx.session.execute(sel).fetchall() - # Create a set of all traits in the db that are not custom - # traits. - db_traits = set( - r[0] for r in res - if not os_traits.is_custom(r[0]) - ) - # Determine those traits which are in os_traits but not - # currently in the database, and insert them. - need_sync = std_traits - db_traits - ins = _TRAIT_TBL.insert() - batch_args = [ - {'name': six.text_type(trait)} - for trait in need_sync - ] - if batch_args: - try: - ctx.session.execute(ins, batch_args) - LOG.info("Synced traits from os_traits into API DB: %s", - need_sync) - except db_exc.DBDuplicateEntry: - pass # some other process sync'd, just ignore - - -def ensure_trait_sync(ctx): - """Ensures that the os_traits library is synchronized to the traits db. - - If _TRAITS_SYNCED is False then this process has not tried to update the - traits db. Do so by calling _trait_sync. Since the placement API server - could be multi-threaded, lock around testing _TRAITS_SYNCED to avoid - duplicating work. - - Different placement API server processes that talk to the same database - will avoid issues through the power of transactions. - - :param ctx: `nova.context.RequestContext` that may be used to grab a DB - connection. - """ - global _TRAITS_SYNCED - # If another thread is doing this work, wait for it to complete. - # When that thread is done _TRAITS_SYNCED will be true in this - # thread and we'll simply return. - with lockutils.lock(_TRAIT_LOCK): - if not _TRAITS_SYNCED: - _trait_sync(ctx) - _TRAITS_SYNCED = True - - -def _get_current_inventory_resources(ctx, rp): - """Returns a set() containing the resource class IDs for all resources - currently having an inventory record for the supplied resource provider. - - :param ctx: `nova.context.RequestContext` that may be used to grab a DB - connection. - :param rp: Resource provider to query inventory for. - """ - cur_res_sel = sa.select([_INV_TBL.c.resource_class_id]).where( - _INV_TBL.c.resource_provider_id == rp.id) - existing_resources = ctx.session.execute(cur_res_sel).fetchall() - return set([r[0] for r in existing_resources]) - - -def _delete_inventory_from_provider(ctx, rp, to_delete): - """Deletes any inventory records from the supplied provider and set() of - resource class identifiers. - - If there are allocations for any of the inventories to be deleted raise - InventoryInUse exception. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param rp: Resource provider from which to delete inventory. - :param to_delete: set() containing resource class IDs for records to - delete. - """ - allocation_query = sa.select( - [_ALLOC_TBL.c.resource_class_id.label('resource_class')]).where( - sa.and_(_ALLOC_TBL.c.resource_provider_id == rp.id, - _ALLOC_TBL.c.resource_class_id.in_(to_delete)) - ).group_by(_ALLOC_TBL.c.resource_class_id) - allocations = ctx.session.execute(allocation_query).fetchall() - if allocations: - resource_classes = ', '.join([_RC_CACHE.string_from_id(alloc[0]) - for alloc in allocations]) - raise exception.InventoryInUse(resource_classes=resource_classes, - resource_provider=rp.uuid) - - del_stmt = _INV_TBL.delete().where(sa.and_( - _INV_TBL.c.resource_provider_id == rp.id, - _INV_TBL.c.resource_class_id.in_(to_delete))) - res = ctx.session.execute(del_stmt) - return res.rowcount - - -def _add_inventory_to_provider(ctx, rp, inv_list, to_add): - """Inserts new inventory records for the supplied resource provider. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param rp: Resource provider to add inventory to. - :param inv_list: InventoryList object - :param to_add: set() containing resource class IDs to search inv_list for - adding to resource provider. - """ - for rc_id in to_add: - rc_str = _RC_CACHE.string_from_id(rc_id) - inv_record = inv_list.find(rc_str) - ins_stmt = _INV_TBL.insert().values( - resource_provider_id=rp.id, - resource_class_id=rc_id, - total=inv_record.total, - reserved=inv_record.reserved, - min_unit=inv_record.min_unit, - max_unit=inv_record.max_unit, - step_size=inv_record.step_size, - allocation_ratio=inv_record.allocation_ratio) - ctx.session.execute(ins_stmt) - - -def _update_inventory_for_provider(ctx, rp, inv_list, to_update): - """Updates existing inventory records for the supplied resource provider. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param rp: Resource provider on which to update inventory. - :param inv_list: InventoryList object - :param to_update: set() containing resource class IDs to search inv_list - for updating in resource provider. - :returns: A list of (uuid, class) tuples that have exceeded their - capacity after this inventory update. - """ - exceeded = [] - for rc_id in to_update: - rc_str = _RC_CACHE.string_from_id(rc_id) - inv_record = inv_list.find(rc_str) - allocation_query = sa.select( - [func.sum(_ALLOC_TBL.c.used).label('usage')]).\ - where(sa.and_( - _ALLOC_TBL.c.resource_provider_id == rp.id, - _ALLOC_TBL.c.resource_class_id == rc_id)) - allocations = ctx.session.execute(allocation_query).first() - if (allocations - and allocations['usage'] is not None - and allocations['usage'] > inv_record.capacity): - exceeded.append((rp.uuid, rc_str)) - upd_stmt = _INV_TBL.update().where(sa.and_( - _INV_TBL.c.resource_provider_id == rp.id, - _INV_TBL.c.resource_class_id == rc_id)).values( - total=inv_record.total, - reserved=inv_record.reserved, - min_unit=inv_record.min_unit, - max_unit=inv_record.max_unit, - step_size=inv_record.step_size, - allocation_ratio=inv_record.allocation_ratio) - res = ctx.session.execute(upd_stmt) - if not res.rowcount: - raise exception.InventoryWithResourceClassNotFound( - resource_class=rc_str) - return exceeded - - -def _increment_provider_generation(ctx, rp): - """Increments the supplied provider's generation value, supplying the - currently-known generation. Returns whether the increment succeeded. - - :param ctx: `nova.context.RequestContext` that contains an oslo_db Session - :param rp: `ResourceProvider` whose generation should be updated. - :returns: The new resource provider generation value if successful. - :raises nova.exception.ConcurrentUpdateDetected: if another thread updated - the same resource provider's view of its inventory or allocations - in between the time when this object was originally read - and the call to set the inventory. - """ - rp_gen = rp.generation - new_generation = rp_gen + 1 - upd_stmt = _RP_TBL.update().where(sa.and_( - _RP_TBL.c.id == rp.id, - _RP_TBL.c.generation == rp_gen)).values( - generation=(new_generation)) - - res = ctx.session.execute(upd_stmt) - if res.rowcount != 1: - raise exception.ResourceProviderConcurrentUpdateDetected() - return new_generation - - -@db_api.placement_context_manager.writer -def _add_inventory(context, rp, inventory): - """Add one Inventory that wasn't already on the provider. - - :raises `exception.ResourceClassNotFound` if inventory.resource_class - cannot be found in either the standard classes or the DB. - """ - rc_id = _RC_CACHE.id_from_string(inventory.resource_class) - inv_list = InventoryList(objects=[inventory]) - _add_inventory_to_provider( - context, rp, inv_list, set([rc_id])) - rp.generation = _increment_provider_generation(context, rp) - - -@db_api.placement_context_manager.writer -def _update_inventory(context, rp, inventory): - """Update an inventory already on the provider. - - :raises `exception.ResourceClassNotFound` if inventory.resource_class - cannot be found in either the standard classes or the DB. - """ - rc_id = _RC_CACHE.id_from_string(inventory.resource_class) - inv_list = InventoryList(objects=[inventory]) - exceeded = _update_inventory_for_provider( - context, rp, inv_list, set([rc_id])) - rp.generation = _increment_provider_generation(context, rp) - return exceeded - - -@db_api.placement_context_manager.writer -def _delete_inventory(context, rp, resource_class): - """Delete up to one Inventory of the given resource_class string. - - :raises `exception.ResourceClassNotFound` if resource_class - cannot be found in either the standard classes or the DB. - """ - rc_id = _RC_CACHE.id_from_string(resource_class) - if not _delete_inventory_from_provider(context, rp, [rc_id]): - raise exception.NotFound( - 'No inventory of class %s found for delete' - % resource_class) - rp.generation = _increment_provider_generation(context, rp) - - -@db_api.placement_context_manager.writer -def _set_inventory(context, rp, inv_list): - """Given an InventoryList object, replaces the inventory of the - resource provider in a safe, atomic fashion using the resource - provider's generation as a consistent view marker. - - :param context: Nova RequestContext. - :param rp: `ResourceProvider` object upon which to set inventory. - :param inv_list: `InventoryList` object to save to backend storage. - :returns: A list of (uuid, class) tuples that have exceeded their - capacity after this inventory update. - :raises nova.exception.ConcurrentUpdateDetected: if another thread updated - the same resource provider's view of its inventory or allocations - in between the time when this object was originally read - and the call to set the inventory. - :raises `exception.ResourceClassNotFound` if any resource class in any - inventory in inv_list cannot be found in either the standard - classes or the DB. - :raises `exception.InventoryInUse` if we attempt to delete inventory - from a provider that has allocations for that resource class. - """ - existing_resources = _get_current_inventory_resources(context, rp) - these_resources = set([_RC_CACHE.id_from_string(r.resource_class) - for r in inv_list.objects]) - - # Determine which resources we should be adding, deleting and/or - # updating in the resource provider's inventory by comparing sets - # of resource class identifiers. - to_add = these_resources - existing_resources - to_delete = existing_resources - these_resources - to_update = these_resources & existing_resources - exceeded = [] - - if to_delete: - _delete_inventory_from_provider(context, rp, to_delete) - if to_add: - _add_inventory_to_provider(context, rp, inv_list, to_add) - if to_update: - exceeded = _update_inventory_for_provider(context, rp, inv_list, - to_update) - - # Here is where we update the resource provider's generation value. If - # this update updates zero rows, that means that another thread has updated - # the inventory for this resource provider between the time the caller - # originally read the resource provider record and inventory information - # and this point. We raise an exception here which will rollback the above - # transaction and return an error to the caller to indicate that they can - # attempt to retry the inventory save after reverifying any capacity - # conditions and re-reading the existing inventory information. - rp.generation = _increment_provider_generation(context, rp) - - return exceeded - - -@db_api.placement_context_manager.reader -def _get_provider_by_uuid(context, uuid): - """Given a UUID, return a dict of information about the resource provider - from the database. - - :raises: NotFound if no such provider was found - :param uuid: The UUID to look up - """ - rpt = sa.alias(_RP_TBL, name="rp") - parent = sa.alias(_RP_TBL, name="parent") - root = sa.alias(_RP_TBL, name="root") - # TODO(jaypipes): Change this to an inner join when we are sure all - # root_provider_id values are NOT NULL - rp_to_root = sa.outerjoin(rpt, root, rpt.c.root_provider_id == root.c.id) - rp_to_parent = sa.outerjoin(rp_to_root, parent, - rpt.c.parent_provider_id == parent.c.id) - cols = [ - rpt.c.id, - rpt.c.uuid, - rpt.c.name, - rpt.c.generation, - root.c.uuid.label("root_provider_uuid"), - parent.c.uuid.label("parent_provider_uuid"), - rpt.c.updated_at, - rpt.c.created_at, - ] - sel = sa.select(cols).select_from(rp_to_parent).where(rpt.c.uuid == uuid) - res = context.session.execute(sel).fetchone() - if not res: - raise exception.NotFound( - 'No resource provider with uuid %s found' % uuid) - return dict(res) - - -@db_api.placement_context_manager.reader -def _get_aggregates_by_provider_id(context, rp_id): - """Returns a dict, keyed by internal aggregate ID, of aggregate UUIDs - associated with the supplied internal resource provider ID. - """ - join_statement = sa.join( - _AGG_TBL, _RP_AGG_TBL, sa.and_( - _AGG_TBL.c.id == _RP_AGG_TBL.c.aggregate_id, - _RP_AGG_TBL.c.resource_provider_id == rp_id)) - sel = sa.select([_AGG_TBL.c.id, _AGG_TBL.c.uuid]).select_from( - join_statement) - return {r[0]: r[1] for r in context.session.execute(sel).fetchall()} - - -@db_api.placement_context_manager.reader -def _anchors_for_sharing_providers(context, rp_ids, get_id=False): - """Given a list of internal IDs of sharing providers, returns a set of - tuples of (sharing provider UUID, anchor provider UUID), where each of - anchor is the unique root provider of a tree associated with the same - aggregate as the sharing provider. (These are the providers that can - "anchor" a single AllocationRequest.) - - The sharing provider may or may not itself be part of a tree; in either - case, an entry for this root provider is included in the result. - - If the sharing provider is not part of any aggregate, the empty list is - returned. - - If get_id is True, it returns a set of tuples of (sharing provider ID, - anchor provider ID) instead. - """ - # SELECT sps.uuid, COALESCE(rps.uuid, shr_with_sps.uuid) - # FROM resource_providers AS sps - # INNER JOIN resource_provider_aggregates AS shr_aggs - # ON sps.id = shr_aggs.resource_provider_id - # INNER JOIN resource_provider_aggregates AS shr_with_sps_aggs - # ON shr_aggs.aggregate_id = shr_with_sps_aggs.aggregate_id - # INNER JOIN resource_providers AS shr_with_sps - # ON shr_with_sps_aggs.resource_provider_id = shr_with_sps.id - # LEFT JOIN resource_providers AS rps - # ON shr_with_sps.root_provider_id = rps.id - # WHERE sps.id IN $(RP_IDs) - rps = sa.alias(_RP_TBL, name='rps') - sps = sa.alias(_RP_TBL, name='sps') - shr_aggs = sa.alias(_RP_AGG_TBL, name='shr_aggs') - shr_with_sps_aggs = sa.alias(_RP_AGG_TBL, name='shr_with_sps_aggs') - shr_with_sps = sa.alias(_RP_TBL, name='shr_with_sps') - join_chain = sa.join( - sps, shr_aggs, sps.c.id == shr_aggs.c.resource_provider_id) - join_chain = sa.join( - join_chain, shr_with_sps_aggs, - shr_aggs.c.aggregate_id == shr_with_sps_aggs.c.aggregate_id) - join_chain = sa.join( - join_chain, shr_with_sps, - shr_with_sps_aggs.c.resource_provider_id == shr_with_sps.c.id) - if get_id: - # TODO(yikun): Change `func.coalesce(shr_with_sps.c.root_provider_id, - # shr_with_sps.c.id)` to `shr_with_sps.c.root_provider_id` when we are - # sure all root_provider_id values are NOT NULL - sel = sa.select([sps.c.id, func.coalesce( - shr_with_sps.c.root_provider_id, shr_with_sps.c.id)]) - else: - # TODO(efried): Change this to an inner join and change - # 'func.coalesce(rps.c.uuid, shr_with_sps.c.uuid)' to `rps.c.uuid` - # when we are sure all root_provider_id values are NOT NULL - join_chain = sa.outerjoin( - join_chain, rps, shr_with_sps.c.root_provider_id == rps.c.id) - sel = sa.select([sps.c.uuid, func.coalesce(rps.c.uuid, - shr_with_sps.c.uuid)]) - sel = sel.select_from(join_chain) - sel = sel.where(sps.c.id.in_(rp_ids)) - return set([(r[0], r[1]) for r in context.session.execute(sel).fetchall()]) - - -@db_api.placement_context_manager.writer -def _ensure_aggregate(ctx, agg_uuid): - """Finds an aggregate and returns its internal ID. If not found, creates - the aggregate and returns the new aggregate's internal ID. - """ - sel = sa.select([_AGG_TBL.c.id]).where(_AGG_TBL.c.uuid == agg_uuid) - res = ctx.session.execute(sel).fetchone() - if res: - return res[0] - - LOG.debug("_ensure_aggregate() did not find aggregate %s. " - "Creating it.", agg_uuid) - try: - ins_stmt = _AGG_TBL.insert().values(uuid=agg_uuid) - res = ctx.session.execute(ins_stmt) - agg_id = res.inserted_primary_key[0] - LOG.debug("_ensure_aggregate() created new aggregate %s (id=%d).", - agg_uuid, agg_id) - return agg_id - except db_exc.DBDuplicateEntry: - # Something else added this agg_uuid in between our initial - # fetch above and when we tried flushing this session, so let's - # grab whatever that other thing added. - LOG.debug("_ensure_provider() failed to create new aggregate %s. " - "Another thread already created an aggregate record. " - "Looking up that aggregate record.", - agg_uuid) - return _ensure_aggregate(ctx, agg_uuid) - - -@db_api.placement_context_manager.writer -def _set_aggregates(context, resource_provider, provided_aggregates, - increment_generation=False): - rp_id = resource_provider.id - # When aggregate uuids are persisted no validation is done - # to ensure that they refer to something that has meaning - # elsewhere. It is assumed that code which makes use of the - # aggregates, later, will validate their fitness. - # TODO(cdent): At the moment we do not delete - # a PlacementAggregate that no longer has any associations - # with at least one resource provider. We may wish to do that - # to avoid bloat if it turns out we're creating a lot of noise. - # Not doing now to move things along. - provided_aggregates = set(provided_aggregates) - existing_aggregates = _get_aggregates_by_provider_id(context, rp_id) - agg_uuids_to_add = provided_aggregates - set(existing_aggregates.values()) - # A dict, keyed by internal aggregate ID, of aggregate UUIDs that will be - # associated with the provider - aggs_to_associate = {} - # Same dict for those aggregates to remove the association with this - # provider - aggs_to_disassociate = { - agg_id: agg_uuid for agg_id, agg_uuid in existing_aggregates.items() - if agg_uuid not in provided_aggregates - } - - # Create any aggregates that do not yet exist in - # PlacementAggregates. This is different from - # the set in existing_aggregates; those are aggregates for - # which there are associations for the resource provider - # at rp_id. The following loop checks for the existence of any - # aggregate with the provided uuid. In this way we only - # create a new row in the PlacementAggregate table if the - # aggregate uuid has never been seen before. Code further - # below will update the associations. - for agg_uuid in agg_uuids_to_add: - agg_id = _ensure_aggregate(context, agg_uuid) - aggs_to_associate[agg_id] = agg_uuid - - for agg_id, agg_uuid in aggs_to_associate.items(): - try: - ins_stmt = _RP_AGG_TBL.insert().values( - resource_provider_id=rp_id, aggregate_id=agg_id) - context.session.execute(ins_stmt) - LOG.debug("Setting aggregates for provider %s. Successfully " - "associated aggregate %s.", - resource_provider.uuid, agg_uuid) - except db_exc.DBDuplicateEntry: - LOG.debug("Setting aggregates for provider %s. Another thread " - "already associated aggregate %s. Skipping.", - resource_provider.uuid, agg_uuid) - pass - - for agg_id, agg_uuid in aggs_to_disassociate.items(): - del_stmt = _RP_AGG_TBL.delete().where(sa.and_( - _RP_AGG_TBL.c.resource_provider_id == rp_id, - _RP_AGG_TBL.c.aggregate_id == agg_id)) - context.session.execute(del_stmt) - LOG.debug("Setting aggregates for provider %s. Successfully " - "disassociated aggregate %s.", - resource_provider.uuid, agg_uuid) - - if increment_generation: - resource_provider.generation = _increment_provider_generation( - context, resource_provider) - - -@db_api.placement_context_manager.reader -def _get_traits_by_provider_id(context, rp_id): - t = sa.alias(_TRAIT_TBL, name='t') - rpt = sa.alias(_RP_TRAIT_TBL, name='rpt') - - join_cond = sa.and_(t.c.id == rpt.c.trait_id, - rpt.c.resource_provider_id == rp_id) - join = sa.join(t, rpt, join_cond) - sel = sa.select([t.c.id, t.c.name, - t.c.created_at, t.c.updated_at]).select_from(join) - return [dict(r) for r in context.session.execute(sel).fetchall()] - - -def _add_traits_to_provider(ctx, rp_id, to_add): - """Adds trait associations to the provider with the supplied ID. - - :param ctx: `nova.context.RequestContext` that has an oslo_db Session - :param rp_id: Internal ID of the resource provider on which to add - trait associations - :param to_add: set() containing internal trait IDs for traits to add - """ - for trait_id in to_add: - try: - ins_stmt = _RP_TRAIT_TBL.insert().values( - resource_provider_id=rp_id, - trait_id=trait_id) - ctx.session.execute(ins_stmt) - except db_exc.DBDuplicateEntry: - # Another thread already set this trait for this provider. Ignore - # this for now (but ConcurrentUpdateDetected will end up being - # raised almost assuredly when we go to increment the resource - # provider's generation later, but that's also fine) - pass - - -def _delete_traits_from_provider(ctx, rp_id, to_delete): - """Deletes trait associations from the provider with the supplied ID and - set() of internal trait IDs. - - :param ctx: `nova.context.RequestContext` that has an oslo_db Session - :param rp_id: Internal ID of the resource provider from which to delete - trait associations - :param to_delete: set() containing internal trait IDs for traits to - delete - """ - del_stmt = _RP_TRAIT_TBL.delete().where( - sa.and_( - _RP_TRAIT_TBL.c.resource_provider_id == rp_id, - _RP_TRAIT_TBL.c.trait_id.in_(to_delete))) - ctx.session.execute(del_stmt) - - -@db_api.placement_context_manager.writer -def _set_traits(context, rp, traits): - """Given a ResourceProvider object and a TraitList object, replaces the set - of traits associated with the resource provider. - - :raises: ConcurrentUpdateDetected if the resource provider's traits or - inventory was changed in between the time when we first started to - set traits and the end of this routine. - - :param rp: The ResourceProvider object to set traits against - :param traits: A TraitList object or list of Trait objects - """ - # Get the internal IDs of our existing traits - existing_traits = _get_traits_by_provider_id(context, rp.id) - existing_traits = set(rec['id'] for rec in existing_traits) - want_traits = set(trait.id for trait in traits) - - to_add = want_traits - existing_traits - to_delete = existing_traits - want_traits - - if not to_add and not to_delete: - return - - if to_delete: - _delete_traits_from_provider(context, rp.id, to_delete) - if to_add: - _add_traits_to_provider(context, rp.id, to_add) - rp.generation = _increment_provider_generation(context, rp) - - -@db_api.placement_context_manager.reader -def _has_child_providers(context, rp_id): - """Returns True if the supplied resource provider has any child providers, - False otherwise - """ - child_sel = sa.select([_RP_TBL.c.id]) - child_sel = child_sel.where(_RP_TBL.c.parent_provider_id == rp_id) - child_res = context.session.execute(child_sel.limit(1)).fetchone() - if child_res: - return True - return False - - -@db_api.placement_context_manager.writer -def _set_root_provider_id(context, rp_id, root_id): - """Simply sets the root_provider_id value for a provider identified by - rp_id. Used in online data migration. - - :param rp_id: Internal ID of the provider to update - :param root_id: Value to set root provider to - """ - upd = _RP_TBL.update().where(_RP_TBL.c.id == rp_id) - upd = upd.values(root_provider_id=root_id) - context.session.execute(upd) - - -ProviderIds = collections.namedtuple( - 'ProviderIds', 'id uuid parent_id parent_uuid root_id root_uuid') - - -def _provider_ids_from_rp_ids(context, rp_ids): - """Given an iterable of internal resource provider IDs, returns a dict, - keyed by internal provider Id, of ProviderIds namedtuples describing those - providers. - - :returns: dict, keyed by internal provider Id, of ProviderIds namedtuples - :param rp_ids: iterable of internal provider IDs to look up - """ - # SELECT - # rp.id, rp.uuid, - # parent.id AS parent_id, parent.uuid AS parent_uuid, - # root.id AS root_id, root.uuid AS root_uuid - # FROM resource_providers AS rp - # LEFT JOIN resource_providers AS parent - # ON rp.parent_provider_id = parent.id - # LEFT JOIN resource_providers AS root - # ON rp.root_provider_id = root.id - # WHERE rp.id IN ($rp_ids) - me = sa.alias(_RP_TBL, name="me") - parent = sa.alias(_RP_TBL, name="parent") - root = sa.alias(_RP_TBL, name="root") - cols = [ - me.c.id, - me.c.uuid, - parent.c.id.label('parent_id'), - parent.c.uuid.label('parent_uuid'), - root.c.id.label('root_id'), - root.c.uuid.label('root_uuid'), - ] - # TODO(jaypipes): Change this to an inner join when we are sure all - # root_provider_id values are NOT NULL - me_to_root = sa.outerjoin(me, root, me.c.root_provider_id == root.c.id) - me_to_parent = sa.outerjoin(me_to_root, parent, - me.c.parent_provider_id == parent.c.id) - sel = sa.select(cols).select_from(me_to_parent) - sel = sel.where(me.c.id.in_(rp_ids)) - - ret = {} - for r in context.session.execute(sel): - # Use its id/uuid for the root id/uuid if the root id/uuid is None - # TODO(tetsuro): Remove this to when we are sure all root_provider_id - # values are NOT NULL - d = dict(r) - if d['root_id'] is None: - d['root_id'] = d['id'] - d['root_uuid'] = d['uuid'] - ret[d['id']] = ProviderIds(**d) - return ret - - -def _provider_ids_from_uuid(context, uuid): - """Given the UUID of a resource provider, returns a namedtuple - (ProviderIds) with the internal ID, the UUID, the parent provider's - internal ID, parent provider's UUID, the root provider's internal ID and - the root provider UUID. - - :returns: ProviderIds object containing the internal IDs and UUIDs of the - provider identified by the supplied UUID - :param uuid: The UUID of the provider to look up - """ - # SELECT - # rp.id, rp.uuid, - # parent.id AS parent_id, parent.uuid AS parent_uuid, - # root.id AS root_id, root.uuid AS root_uuid - # FROM resource_providers AS rp - # LEFT JOIN resource_providers AS parent - # ON rp.parent_provider_id = parent.id - # LEFT JOIN resource_providers AS root - # ON rp.root_provider_id = root.id - me = sa.alias(_RP_TBL, name="me") - parent = sa.alias(_RP_TBL, name="parent") - root = sa.alias(_RP_TBL, name="root") - cols = [ - me.c.id, - me.c.uuid, - parent.c.id.label('parent_id'), - parent.c.uuid.label('parent_uuid'), - root.c.id.label('root_id'), - root.c.uuid.label('root_uuid'), - ] - # TODO(jaypipes): Change this to an inner join when we are sure all - # root_provider_id values are NOT NULL - me_to_root = sa.outerjoin(me, root, me.c.root_provider_id == root.c.id) - me_to_parent = sa.outerjoin(me_to_root, parent, - me.c.parent_provider_id == parent.c.id) - sel = sa.select(cols).select_from(me_to_parent) - sel = sel.where(me.c.uuid == uuid) - res = context.session.execute(sel).fetchone() - if not res: - return None - return ProviderIds(**dict(res)) - - -def _provider_ids_matching_aggregates(context, member_of, rp_ids=None): - """Given a list of lists of aggregate UUIDs, return the internal IDs of all - resource providers associated with the aggregates. - - :param member_of: A list containing lists of aggregate UUIDs. Each item in - the outer list is to be AND'd together. If that item contains multiple - values, they are OR'd together. - - For example, if member_of is:: - - [ - ['agg1'], - ['agg2', 'agg3'], - ] - - we will return all the resource providers that are - associated with agg1 as well as either (agg2 or agg3) - :param rp_ids: When present, returned resource providers are limited - to only those in this value - - :returns: A set of internal resource provider IDs having all required - aggregate associations - """ - # Given a request for the following: - # - # member_of = [ - # [agg1], - # [agg2], - # [agg3, agg4] - # ] - # - # we need to produce the following SQL expression: - # - # SELECT - # rp.id - # FROM resource_providers AS rp - # JOIN resource_provider_aggregates AS rpa1 - # ON rp.id = rpa1.resource_provider_id - # AND rpa1.aggregate_id IN ($AGG1_ID) - # JOIN resource_provider_aggregates AS rpa2 - # ON rp.id = rpa2.resource_provider_id - # AND rpa2.aggregate_id IN ($AGG2_ID) - # JOIN resource_provider_aggregates AS rpa3 - # ON rp.id = rpa3.resource_provider_id - # AND rpa3.aggregate_id IN ($AGG3_ID, $AGG4_ID) - # # Only if we have rp_ids... - # WHERE rp.id IN ($RP_IDs) - - # First things first, get a map of all the aggregate UUID to internal - # aggregate IDs - agg_uuids = set() - for members in member_of: - for member in members: - agg_uuids.add(member) - agg_tbl = sa.alias(_AGG_TBL, name='aggs') - agg_sel = sa.select([agg_tbl.c.uuid, agg_tbl.c.id]) - agg_sel = agg_sel.where(agg_tbl.c.uuid.in_(agg_uuids)) - agg_uuid_map = { - r[0]: r[1] for r in context.session.execute(agg_sel).fetchall() - } - - rp_tbl = sa.alias(_RP_TBL, name='rp') - join_chain = rp_tbl - - for x, members in enumerate(member_of): - rpa_tbl = sa.alias(_RP_AGG_TBL, name='rpa%d' % x) - - agg_ids = [agg_uuid_map[member] for member in members - if member in agg_uuid_map] - if not agg_ids: - # This member_of list contains only non-existent aggregate UUIDs - # and therefore we will always return 0 results, so short-circuit - return [] - - join_cond = sa.and_( - rp_tbl.c.id == rpa_tbl.c.resource_provider_id, - rpa_tbl.c.aggregate_id.in_(agg_ids)) - join_chain = sa.join(join_chain, rpa_tbl, join_cond) - sel = sa.select([rp_tbl.c.id]).select_from(join_chain) - if rp_ids: - sel = sel.where(rp_tbl.c.id.in_(rp_ids)) - return set(r[0] for r in context.session.execute(sel)) - - -@db_api.placement_context_manager.writer -def _delete_rp_record(context, _id): - return context.session.query(models.ResourceProvider).\ - filter(models.ResourceProvider.id == _id).\ - delete(synchronize_session=False) - - -@base.VersionedObjectRegistry.register_if(False) -class ResourceProvider(base.VersionedObject, base.TimestampedObject): - SETTABLE_FIELDS = ('name', 'parent_provider_uuid') - - fields = { - 'id': fields.IntegerField(read_only=True), - 'uuid': fields.UUIDField(nullable=False), - 'name': fields.StringField(nullable=False), - 'generation': fields.IntegerField(nullable=False), - # UUID of the root provider in a hierarchy of providers. Will be equal - # to the uuid field if this provider is the root provider of a - # hierarchy. This field is never manually set by the user. Instead, it - # is automatically set to either the root provider UUID of the parent - # or the UUID of the provider itself if there is no parent. This field - # is an optimization field that allows us to very quickly query for all - # providers within a particular tree without doing any recursive - # querying. - 'root_provider_uuid': fields.UUIDField(nullable=False), - # UUID of the direct parent provider, or None if this provider is a - # "root" provider. - 'parent_provider_uuid': fields.UUIDField(nullable=True, default=None), - } - - def create(self): - if 'id' in self: - raise exception.ObjectActionError(action='create', - reason='already created') - if 'uuid' not in self: - raise exception.ObjectActionError(action='create', - reason='uuid is required') - if 'name' not in self: - raise exception.ObjectActionError(action='create', - reason='name is required') - if 'root_provider_uuid' in self: - raise exception.ObjectActionError( - action='create', - reason=_('root provider UUID cannot be manually set.')) - - self.obj_set_defaults() - updates = self.obj_get_changes() - self._create_in_db(self._context, updates) - self.obj_reset_changes() - - def destroy(self): - self._delete(self._context, self.id) - - def save(self): - updates = self.obj_get_changes() - if updates and any(k not in self.SETTABLE_FIELDS - for k in updates.keys()): - raise exception.ObjectActionError( - action='save', - reason='Immutable fields changed') - self._update_in_db(self._context, self.id, updates) - self.obj_reset_changes() - - @classmethod - def get_by_uuid(cls, context, uuid): - """Returns a new ResourceProvider object with the supplied UUID. - - :raises NotFound if no such provider could be found - :param uuid: UUID of the provider to search for - """ - rp_rec = _get_provider_by_uuid(context, uuid) - return cls._from_db_object(context, cls(), rp_rec) - - def add_inventory(self, inventory): - """Add one new Inventory to the resource provider. - - Fails if Inventory of the provided resource class is - already present. - """ - _add_inventory(self._context, self, inventory) - self.obj_reset_changes() - - def delete_inventory(self, resource_class): - """Delete Inventory of provided resource_class.""" - _delete_inventory(self._context, self, resource_class) - self.obj_reset_changes() - - def set_inventory(self, inv_list): - """Set all resource provider Inventory to be the provided list.""" - exceeded = _set_inventory(self._context, self, inv_list) - for uuid, rclass in exceeded: - LOG.warning('Resource provider %(uuid)s is now over-' - 'capacity for %(resource)s', - {'uuid': uuid, 'resource': rclass}) - self.obj_reset_changes() - - def update_inventory(self, inventory): - """Update one existing Inventory of the same resource class. - - Fails if no Inventory of the same class is present. - """ - exceeded = _update_inventory(self._context, self, inventory) - for uuid, rclass in exceeded: - LOG.warning('Resource provider %(uuid)s is now over-' - 'capacity for %(resource)s', - {'uuid': uuid, 'resource': rclass}) - self.obj_reset_changes() - - def get_aggregates(self): - """Get the aggregate uuids associated with this resource provider.""" - return list( - _get_aggregates_by_provider_id(self._context, self.id).values()) - - def set_aggregates(self, aggregate_uuids, increment_generation=False): - """Set the aggregate uuids associated with this resource provider. - - If an aggregate does not exist, one will be created using the - provided uuid. - - The resource provider generation is incremented if and only if the - increment_generation parameter is True. - """ - _set_aggregates(self._context, self, aggregate_uuids, - increment_generation=increment_generation) - - def set_traits(self, traits): - """Replaces the set of traits associated with the resource provider - with the given list of Trait objects. - - :param traits: A list of Trait objects representing the traits to - associate with the provider. - """ - _set_traits(self._context, self, traits) - self.obj_reset_changes() - - @db_api.placement_context_manager.writer - def _create_in_db(self, context, updates): - parent_id = None - root_id = None - # User supplied a parent, let's make sure it exists - parent_uuid = updates.pop('parent_provider_uuid') - if parent_uuid is not None: - # Setting parent to ourselves doesn't make any sense - if parent_uuid == self.uuid: - raise exception.ObjectActionError( - action='create', - reason=_('parent provider UUID cannot be same as ' - 'UUID. Please set parent provider UUID to ' - 'None if there is no parent.')) - - parent_ids = _provider_ids_from_uuid(context, parent_uuid) - if parent_ids is None: - raise exception.ObjectActionError( - action='create', - reason=_('parent provider UUID does not exist.')) - - parent_id = parent_ids.id - root_id = parent_ids.root_id - updates['root_provider_id'] = root_id - updates['parent_provider_id'] = parent_id - self.root_provider_uuid = parent_ids.root_uuid - - db_rp = models.ResourceProvider() - db_rp.update(updates) - context.session.add(db_rp) - context.session.flush() - - self.id = db_rp.id - self.generation = db_rp.generation - - if root_id is None: - # User did not specify a parent when creating this provider, so the - # root_provider_id needs to be set to this provider's newly-created - # internal ID - db_rp.root_provider_id = db_rp.id - context.session.add(db_rp) - context.session.flush() - self.root_provider_uuid = self.uuid - - @staticmethod - @db_api.placement_context_manager.writer - def _delete(context, _id): - # Do a quick check to see if the provider is a parent. If it is, don't - # allow deleting the provider. Note that the foreign key constraint on - # resource_providers.parent_provider_id will prevent deletion of the - # parent within the transaction below. This is just a quick - # short-circuit outside of the transaction boundary. - if _has_child_providers(context, _id): - raise exception.CannotDeleteParentResourceProvider() - - # Don't delete the resource provider if it has allocations. - rp_allocations = context.session.query(models.Allocation).\ - filter(models.Allocation.resource_provider_id == _id).\ - count() - if rp_allocations: - raise exception.ResourceProviderInUse() - # Delete any inventory associated with the resource provider - context.session.query(models.Inventory).\ - filter(models.Inventory.resource_provider_id == _id).\ - delete(synchronize_session=False) - # Delete any aggregate associations for the resource provider - # The name substitution on the next line is needed to satisfy pep8 - RPA_model = models.ResourceProviderAggregate - context.session.query(RPA_model).\ - filter(RPA_model.resource_provider_id == _id).delete() - # delete any trait associations for the resource provider - RPT_model = models.ResourceProviderTrait - context.session.query(RPT_model).\ - filter(RPT_model.resource_provider_id == _id).delete() - # set root_provider_id to null to make deletion possible - context.session.query(models.ResourceProvider).\ - filter(models.ResourceProvider.id == _id, - models.ResourceProvider.root_provider_id == _id).\ - update({'root_provider_id': None}) - # Now delete the RP record - try: - result = _delete_rp_record(context, _id) - except sqla_exc.IntegrityError: - # NOTE(jaypipes): Another thread snuck in and parented this - # resource provider in between the above check for - # _has_child_providers() and our attempt to delete the record - raise exception.CannotDeleteParentResourceProvider() - if not result: - raise exception.NotFound() - - @db_api.placement_context_manager.writer - def _update_in_db(self, context, id, updates): - # A list of resource providers in the same tree with the - # resource provider to update - same_tree = [] - if 'parent_provider_uuid' in updates: - # TODO(jaypipes): For now, "re-parenting" and "un-parenting" are - # not possible. If the provider already had a parent, we don't - # allow changing that parent due to various issues, including: - # - # * if the new parent is a descendant of this resource provider, we - # introduce the possibility of a loop in the graph, which would - # be very bad - # * potentially orphaning heretofore-descendants - # - # So, for now, let's just prevent re-parenting... - my_ids = _provider_ids_from_uuid(context, self.uuid) - parent_uuid = updates.pop('parent_provider_uuid') - if parent_uuid is not None: - parent_ids = _provider_ids_from_uuid(context, parent_uuid) - # User supplied a parent, let's make sure it exists - if parent_ids is None: - raise exception.ObjectActionError( - action='create', - reason=_('parent provider UUID does not exist.')) - if (my_ids.parent_id is not None and - my_ids.parent_id != parent_ids.id): - raise exception.ObjectActionError( - action='update', - reason=_('re-parenting a provider is not ' - 'currently allowed.')) - if my_ids.parent_uuid is None: - # So the user specifies a parent for an RP that doesn't - # have one. We have to check that by this new parent we - # don't create a loop in the tree. Basically the new parent - # cannot be the RP itself or one of its descendants. - # However as the RP's current parent is None the above - # condition is the same as "the new parent cannot be any RP - # from the current RP tree". - same_tree = ResourceProviderList.get_all_by_filters( - context, - filters={'in_tree': self.uuid}) - rp_uuids_in_the_same_tree = [rp.uuid for rp in same_tree] - if parent_uuid in rp_uuids_in_the_same_tree: - raise exception.ObjectActionError( - action='update', - reason=_('creating loop in the provider tree is ' - 'not allowed.')) - - updates['root_provider_id'] = parent_ids.root_id - updates['parent_provider_id'] = parent_ids.id - self.root_provider_uuid = parent_ids.root_uuid - else: - if my_ids.parent_id is not None: - raise exception.ObjectActionError( - action='update', - reason=_('un-parenting a provider is not ' - 'currently allowed.')) - - db_rp = context.session.query(models.ResourceProvider).filter_by( - id=id).first() - db_rp.update(updates) - context.session.add(db_rp) - - # We should also update the root providers of resource providers - # originally in the same tree. If re-parenting is supported, - # this logic should be changed to update only descendents of the - # re-parented resource providers, not all the providers in the tree. - for rp in same_tree: - # If the parent is not updated, this clause is skipped since the - # `same_tree` has no element. - rp.root_provider_uuid = parent_ids.root_uuid - db_rp = context.session.query( - models.ResourceProvider).filter_by(id=rp.id).first() - data = {'root_provider_id': parent_ids.root_id} - db_rp.update(data) - context.session.add(db_rp) - - try: - context.session.flush() - except sqla_exc.IntegrityError: - # NOTE(jaypipes): Another thread snuck in and deleted the parent - # for this resource provider in between the above check for a valid - # parent provider and here... - raise exception.ObjectActionError( - action='update', - reason=_('parent provider UUID does not exist.')) - - @staticmethod - @db_api.placement_context_manager.writer # For online data migration - def _from_db_object(context, resource_provider, db_resource_provider): - # Online data migration to populate root_provider_id - # TODO(jaypipes): Remove when all root_provider_id values are NOT NULL - if db_resource_provider['root_provider_uuid'] is None: - rp_id = db_resource_provider['id'] - uuid = db_resource_provider['uuid'] - db_resource_provider['root_provider_uuid'] = uuid - _set_root_provider_id(context, rp_id, rp_id) - for field in resource_provider.fields: - setattr(resource_provider, field, db_resource_provider[field]) - resource_provider._context = context - resource_provider.obj_reset_changes() - return resource_provider - - -@db_api.placement_context_manager.reader -def _get_providers_with_shared_capacity(ctx, rc_id, amount, member_of=None): - """Returns a list of resource provider IDs (internal IDs, not UUIDs) - that have capacity for a requested amount of a resource and indicate that - they share resource via an aggregate association. - - Shared resource providers are marked with a standard trait called - MISC_SHARES_VIA_AGGREGATE. This indicates that the provider allows its - inventory to be consumed by other resource providers associated via an - aggregate link. - - For example, assume we have two compute nodes, CN_1 and CN_2, each with - inventory of VCPU and MEMORY_MB but not DISK_GB (in other words, these are - compute nodes with no local disk). There is a resource provider called - "NFS_SHARE" that has an inventory of DISK_GB and has the - MISC_SHARES_VIA_AGGREGATE trait. Both the "CN_1" and "CN_2" compute node - resource providers and the "NFS_SHARE" resource provider are associated - with an aggregate called "AGG_1". - - The scheduler needs to determine the resource providers that can fulfill a - request for 2 VCPU, 1024 MEMORY_MB and 100 DISK_GB. - - Clearly, no single provider can satisfy the request for all three - resources, since neither compute node has DISK_GB inventory and the - NFS_SHARE provider has no VCPU or MEMORY_MB inventories. - - However, if we consider the NFS_SHARE resource provider as providing - inventory of DISK_GB for both CN_1 and CN_2, we can include CN_1 and CN_2 - as potential fits for the requested set of resources. - - To facilitate that matching query, this function returns all providers that - indicate they share their inventory with providers in some aggregate and - have enough capacity for the requested amount of a resource. - - To follow the example above, if we were to call - _get_providers_with_shared_capacity(ctx, "DISK_GB", 100), we would want to - get back the ID for the NFS_SHARE resource provider. - - :param rc_id: Internal ID of the requested resource class. - :param amount: Amount of the requested resource. - :param member_of: When present, contains a list of lists of aggregate - uuids that are used to filter the returned list of - resource providers that *directly* belong to the - aggregates referenced. - """ - # The SQL we need to generate here looks like this: - # - # SELECT rp.id - # FROM resource_providers AS rp - # INNER JOIN resource_provider_traits AS rpt - # ON rp.id = rpt.resource_provider_id - # INNER JOIN traits AS t - # ON rpt.trait_id = t.id - # AND t.name = "MISC_SHARES_VIA_AGGREGATE" - # INNER JOIN inventories AS inv - # ON rp.id = inv.resource_provider_id - # AND inv.resource_class_id = $rc_id - # LEFT JOIN ( - # SELECT resource_provider_id, SUM(used) as used - # FROM allocations - # WHERE resource_class_id = $rc_id - # GROUP BY resource_provider_id - # ) AS usage - # ON rp.id = usage.resource_provider_id - # WHERE COALESCE(usage.used, 0) + $amount <= ( - # inv.total - inv.reserved) * inv.allocation_ratio - # ) AND - # inv.min_unit <= $amount AND - # inv.max_unit >= $amount AND - # $amount % inv.step_size = 0 - # GROUP BY rp.id - - rp_tbl = sa.alias(_RP_TBL, name='rp') - inv_tbl = sa.alias(_INV_TBL, name='inv') - t_tbl = sa.alias(_TRAIT_TBL, name='t') - rpt_tbl = sa.alias(_RP_TRAIT_TBL, name='rpt') - - rp_to_rpt_join = sa.join( - rp_tbl, rpt_tbl, - rp_tbl.c.id == rpt_tbl.c.resource_provider_id, - ) - - rpt_to_t_join = sa.join( - rp_to_rpt_join, t_tbl, - sa.and_( - rpt_tbl.c.trait_id == t_tbl.c.id, - # The traits table wants unicode trait names, but os_traits - # presents native str, so we need to cast. - t_tbl.c.name == six.text_type(os_traits.MISC_SHARES_VIA_AGGREGATE), - ), - ) - - rp_to_inv_join = sa.join( - rpt_to_t_join, inv_tbl, - sa.and_( - rpt_tbl.c.resource_provider_id == inv_tbl.c.resource_provider_id, - inv_tbl.c.resource_class_id == rc_id, - ), - ) - - usage = sa.select([_ALLOC_TBL.c.resource_provider_id, - sql.func.sum(_ALLOC_TBL.c.used).label('used')]) - usage = usage.where(_ALLOC_TBL.c.resource_class_id == rc_id) - usage = usage.group_by(_ALLOC_TBL.c.resource_provider_id) - usage = sa.alias(usage, name='usage') - - inv_to_usage_join = sa.outerjoin( - rp_to_inv_join, usage, - inv_tbl.c.resource_provider_id == usage.c.resource_provider_id, - ) - - where_conds = sa.and_( - func.coalesce(usage.c.used, 0) + amount <= ( - inv_tbl.c.total - inv_tbl.c.reserved) * inv_tbl.c.allocation_ratio, - inv_tbl.c.min_unit <= amount, - inv_tbl.c.max_unit >= amount, - amount % inv_tbl.c.step_size == 0) - - # If 'member_of' has values, do a separate lookup to identify the - # resource providers that meet the member_of constraints. - if member_of: - rps_in_aggs = _provider_ids_matching_aggregates(ctx, member_of) - if not rps_in_aggs: - # Short-circuit. The user either asked for a non-existing - # aggregate or there were no resource providers that matched - # the requirements... - return [] - where_conds.append(rp_tbl.c.id.in_(rps_in_aggs)) - - sel = sa.select([rp_tbl.c.id]).select_from(inv_to_usage_join) - sel = sel.where(where_conds) - sel = sel.group_by(rp_tbl.c.id) - - return [r[0] for r in ctx.session.execute(sel)] - - -@base.VersionedObjectRegistry.register_if(False) -class ResourceProviderList(base.ObjectListBase, base.VersionedObject): - - fields = { - 'objects': fields.ListOfObjectsField('ResourceProvider'), - } - - @staticmethod - @db_api.placement_context_manager.reader - def _get_all_by_filters_from_db(context, filters): - # Eg. filters can be: - # filters = { - # 'name': , - # 'uuid': , - # 'member_of': [[, ], - # []] - # 'resources': { - # 'VCPU': 1, - # 'MEMORY_MB': 1024 - # }, - # 'in_tree': , - # 'required': [, ...] - # } - if not filters: - filters = {} - else: - # Since we modify the filters, copy them so that we don't modify - # them in the calling program. - filters = copy.deepcopy(filters) - name = filters.pop('name', None) - uuid = filters.pop('uuid', None) - member_of = filters.pop('member_of', []) - required = set(filters.pop('required', [])) - forbidden = set([trait for trait in required - if trait.startswith('!')]) - required = required - forbidden - forbidden = set([trait.lstrip('!') for trait in forbidden]) - - resources = filters.pop('resources', {}) - # NOTE(sbauza): We want to key the dict by the resource class IDs - # and we want to make sure those class names aren't incorrect. - resources = {_RC_CACHE.id_from_string(r_name): amount - for r_name, amount in resources.items()} - rp = sa.alias(_RP_TBL, name="rp") - root_rp = sa.alias(_RP_TBL, name="root_rp") - parent_rp = sa.alias(_RP_TBL, name="parent_rp") - - cols = [ - rp.c.id, - rp.c.uuid, - rp.c.name, - rp.c.generation, - rp.c.updated_at, - rp.c.created_at, - root_rp.c.uuid.label("root_provider_uuid"), - parent_rp.c.uuid.label("parent_provider_uuid"), - ] - - # TODO(jaypipes): Convert this to an inner join once all - # root_provider_id values are NOT NULL - rp_to_root = sa.outerjoin(rp, root_rp, - rp.c.root_provider_id == root_rp.c.id) - rp_to_parent = sa.outerjoin(rp_to_root, parent_rp, - rp.c.parent_provider_id == parent_rp.c.id) - - query = sa.select(cols).select_from(rp_to_parent) - - if name: - query = query.where(rp.c.name == name) - if uuid: - query = query.where(rp.c.uuid == uuid) - if 'in_tree' in filters: - # The 'in_tree' parameter is the UUID of a resource provider that - # the caller wants to limit the returned providers to only those - # within its "provider tree". So, we look up the resource provider - # having the UUID specified by the 'in_tree' parameter and grab the - # root_provider_id value of that record. We can then ask for only - # those resource providers having a root_provider_id of that value. - tree_uuid = filters.pop('in_tree') - tree_ids = _provider_ids_from_uuid(context, tree_uuid) - if tree_ids is None: - # List operations should simply return an empty list when a - # non-existing resource provider UUID is given. - return [] - root_id = tree_ids.root_id - # TODO(jaypipes): Remove this OR condition when root_provider_id - # is not nullable in the database and all resource provider records - # have populated the root provider ID. - where_cond = sa.or_(rp.c.id == root_id, - rp.c.root_provider_id == root_id) - query = query.where(where_cond) - - # If 'member_of' has values, do a separate lookup to identify the - # resource providers that meet the member_of constraints. - if member_of: - rps_in_aggs = _provider_ids_matching_aggregates(context, member_of) - if not rps_in_aggs: - # Short-circuit. The user either asked for a non-existing - # aggregate or there were no resource providers that matched - # the requirements... - return [] - query = query.where(rp.c.id.in_(rps_in_aggs)) - - # If 'required' has values, add a filter to limit results to providers - # possessing *all* of the listed traits. - if required: - trait_map = _trait_ids_from_names(context, required) - if len(trait_map) != len(required): - missing = required - set(trait_map) - raise exception.TraitNotFound(names=', '.join(missing)) - rp_ids = _get_provider_ids_having_all_traits(context, trait_map) - if not rp_ids: - # If no providers have the required traits, we're done - return [] - query = query.where(rp.c.id.in_(rp_ids)) - - # If 'forbidden' has values, filter out those providers that have - # that trait as one their traits. - if forbidden: - trait_map = _trait_ids_from_names(context, forbidden) - if len(trait_map) != len(forbidden): - missing = forbidden - set(trait_map) - raise exception.TraitNotFound(names=', '.join(missing)) - rp_ids = _get_provider_ids_having_any_trait(context, trait_map) - if rp_ids: - query = query.where(~rp.c.id.in_(rp_ids)) - - if not resources: - # Returns quickly the list in case we don't need to check the - # resource usage - res = context.session.execute(query).fetchall() - return [dict(r) for r in res] - - # NOTE(sbauza): In case we want to look at the resource criteria, then - # the SQL generated from this case looks something like: - # SELECT - # rp.* - # FROM resource_providers AS rp - # JOIN inventories AS inv - # ON rp.id = inv.resource_provider_id - # LEFT JOIN ( - # SELECT resource_provider_id, resource_class_id, SUM(used) AS used - # FROM allocations - # WHERE resource_class_id IN ($RESOURCE_CLASSES) - # GROUP BY resource_provider_id, resource_class_id - # ) AS usage - # ON inv.resource_provider_id = usage.resource_provider_id - # AND inv.resource_class_id = usage.resource_class_id - # AND (inv.resource_class_id = $X AND (used + $AMOUNT_X <= ( - # total - reserved) * inv.allocation_ratio) AND - # inv.min_unit <= $AMOUNT_X AND inv.max_unit >= $AMOUNT_X AND - # $AMOUNT_X % inv.step_size == 0) - # OR (inv.resource_class_id = $Y AND (used + $AMOUNT_Y <= ( - # total - reserved) * inv.allocation_ratio) AND - # inv.min_unit <= $AMOUNT_Y AND inv.max_unit >= $AMOUNT_Y AND - # $AMOUNT_Y % inv.step_size == 0) - # OR (inv.resource_class_id = $Z AND (used + $AMOUNT_Z <= ( - # total - reserved) * inv.allocation_ratio) AND - # inv.min_unit <= $AMOUNT_Z AND inv.max_unit >= $AMOUNT_Z AND - # $AMOUNT_Z % inv.step_size == 0)) - # GROUP BY rp.id - # HAVING - # COUNT(DISTINCT(inv.resource_class_id)) == len($RESOURCE_CLASSES) - # - # with a possible additional WHERE clause for the name and uuid that - # comes from the above filters - - # First JOIN between inventories and RPs is here - inv_join = sa.join(rp_to_parent, _INV_TBL, - rp.c.id == _INV_TBL.c.resource_provider_id) - - # Now, below is the LEFT JOIN for getting the allocations usage - usage = sa.select([_ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id, - sql.func.sum(_ALLOC_TBL.c.used).label('used')]) - usage = usage.where(_ALLOC_TBL.c.resource_class_id.in_(resources)) - usage = usage.group_by(_ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id) - usage = sa.alias(usage, name='usage') - usage_join = sa.outerjoin(inv_join, usage, - sa.and_( - usage.c.resource_provider_id == ( - _INV_TBL.c.resource_provider_id), - usage.c.resource_class_id == _INV_TBL.c.resource_class_id)) - - # And finally, we verify for each resource class if the requested - # amount isn't more than the left space (considering the allocation - # ratio, the reserved space and the min and max amount possible sizes) - where_clauses = [ - sa.and_( - _INV_TBL.c.resource_class_id == r_idx, - (func.coalesce(usage.c.used, 0) + amount <= ( - _INV_TBL.c.total - _INV_TBL.c.reserved - ) * _INV_TBL.c.allocation_ratio), - _INV_TBL.c.min_unit <= amount, - _INV_TBL.c.max_unit >= amount, - amount % _INV_TBL.c.step_size == 0 - ) - for (r_idx, amount) in resources.items()] - query = query.select_from(usage_join) - query = query.where(sa.or_(*where_clauses)) - query = query.group_by(rp.c.id, root_rp.c.uuid, parent_rp.c.uuid) - # NOTE(sbauza): Only RPs having all the asked resources can be provided - query = query.having(sql.func.count( - sa.distinct(_INV_TBL.c.resource_class_id)) == len(resources)) - - res = context.session.execute(query).fetchall() - return [dict(r) for r in res] - - @classmethod - def get_all_by_filters(cls, context, filters=None): - """Returns a list of `ResourceProvider` objects that have sufficient - resources in their inventories to satisfy the amounts specified in the - `filters` parameter. - - If no resource providers can be found, the function will return an - empty list. - - :param context: `nova.context.RequestContext` that may be used to grab - a DB connection. - :param filters: Can be `name`, `uuid`, `member_of`, `in_tree` or - `resources` where `member_of` is a list of list of - aggregate UUIDs, `in_tree` is a UUID of a resource - provider that we can use to find the root provider ID - of the tree of providers to filter results by and - `resources` is a dict of amounts keyed by resource - classes. - :type filters: dict - """ - resource_providers = cls._get_all_by_filters_from_db(context, filters) - return base.obj_make_list(context, cls(context), - ResourceProvider, resource_providers) - - -@base.VersionedObjectRegistry.register_if(False) -class Inventory(base.VersionedObject, base.TimestampedObject): - - fields = { - 'id': fields.IntegerField(read_only=True), - 'resource_provider': fields.ObjectField('ResourceProvider'), - 'resource_class': rc_fields.ResourceClassField(read_only=True), - 'total': fields.NonNegativeIntegerField(), - 'reserved': fields.NonNegativeIntegerField(default=0), - 'min_unit': fields.NonNegativeIntegerField(default=1), - 'max_unit': fields.NonNegativeIntegerField(default=1), - 'step_size': fields.NonNegativeIntegerField(default=1), - 'allocation_ratio': fields.NonNegativeFloatField(default=1.0), - } - - @property - def capacity(self): - """Inventory capacity, adjusted by allocation_ratio.""" - return int((self.total - self.reserved) * self.allocation_ratio) - - -@db_api.placement_context_manager.reader -def _get_inventory_by_provider_id(ctx, rp_id): - inv = sa.alias(_INV_TBL, name="i") - cols = [ - inv.c.resource_class_id, - inv.c.total, - inv.c.reserved, - inv.c.min_unit, - inv.c.max_unit, - inv.c.step_size, - inv.c.allocation_ratio, - inv.c.updated_at, - inv.c.created_at, - ] - sel = sa.select(cols) - sel = sel.where(inv.c.resource_provider_id == rp_id) - - return [dict(r) for r in ctx.session.execute(sel)] - - -@base.VersionedObjectRegistry.register_if(False) -class InventoryList(base.ObjectListBase, base.VersionedObject): - - fields = { - 'objects': fields.ListOfObjectsField('Inventory'), - } - - def find(self, res_class): - """Return the inventory record from the list of Inventory records that - matches the supplied resource class, or None. - - :param res_class: An integer or string representing a resource - class. If the value is a string, the method first - looks up the resource class identifier from the - string. - """ - if not isinstance(res_class, six.string_types): - raise ValueError - - for inv_rec in self.objects: - if inv_rec.resource_class == res_class: - return inv_rec - - @classmethod - def get_all_by_resource_provider(cls, context, rp): - db_inv = _get_inventory_by_provider_id(context, rp.id) - # Build up a list of Inventory objects, setting the Inventory object - # fields to the same-named database record field we got from - # _get_inventory_by_provider_id(). We already have the ResourceProvider - # object so we just pass that object to the Inventory object - # constructor as-is - objs = [ - Inventory( - context, resource_provider=rp, - resource_class=_RC_CACHE.string_from_id( - rec['resource_class_id']), - **rec) - for rec in db_inv - ] - inv_list = cls(context, objects=objs) - return inv_list - - -@base.VersionedObjectRegistry.register_if(False) -class Allocation(base.VersionedObject, base.TimestampedObject): - - fields = { - 'id': fields.IntegerField(), - 'resource_provider': fields.ObjectField('ResourceProvider'), - 'consumer': fields.ObjectField('Consumer', nullable=False), - 'resource_class': rc_fields.ResourceClassField(), - 'used': fields.IntegerField(), - } - - -@db_api.placement_context_manager.writer -def _delete_allocations_for_consumer(ctx, consumer_id): - """Deletes any existing allocations that correspond to the allocations to - be written. This is wrapped in a transaction, so if the write subsequently - fails, the deletion will also be rolled back. - """ - del_sql = _ALLOC_TBL.delete().where( - _ALLOC_TBL.c.consumer_id == consumer_id) - ctx.session.execute(del_sql) - - -@db_api.placement_context_manager.writer -def _delete_allocations_by_ids(ctx, alloc_ids): - """Deletes allocations having an internal id value in the set of supplied - IDs - """ - del_sql = _ALLOC_TBL.delete().where(_ALLOC_TBL.c.id.in_(alloc_ids)) - ctx.session.execute(del_sql) - - -def _check_capacity_exceeded(ctx, allocs): - """Checks to see if the supplied allocation records would result in any of - the inventories involved having their capacity exceeded. - - Raises an InvalidAllocationCapacityExceeded exception if any inventory - would be exhausted by the allocation. Raises an - InvalidAllocationConstraintsViolated exception if any of the `step_size`, - `min_unit` or `max_unit` constraints in an inventory will be violated - by any one of the allocations. - - If no inventories would be exceeded or violated by the allocations, the - function returns a list of `ResourceProvider` objects that contain the - generation at the time of the check. - - :param ctx: `nova.context.RequestContext` that has an oslo_db Session - :param allocs: List of `Allocation` objects to check - """ - # The SQL generated below looks like this: - # SELECT - # rp.id, - # rp.uuid, - # rp.generation, - # inv.resource_class_id, - # inv.total, - # inv.reserved, - # inv.allocation_ratio, - # allocs.used - # FROM resource_providers AS rp - # JOIN inventories AS i1 - # ON rp.id = i1.resource_provider_id - # LEFT JOIN ( - # SELECT resource_provider_id, resource_class_id, SUM(used) AS used - # FROM allocations - # WHERE resource_class_id IN ($RESOURCE_CLASSES) - # AND resource_provider_id IN ($RESOURCE_PROVIDERS) - # GROUP BY resource_provider_id, resource_class_id - # ) AS allocs - # ON inv.resource_provider_id = allocs.resource_provider_id - # AND inv.resource_class_id = allocs.resource_class_id - # WHERE rp.id IN ($RESOURCE_PROVIDERS) - # AND inv.resource_class_id IN ($RESOURCE_CLASSES) - # - # We then take the results of the above and determine if any of the - # inventory will have its capacity exceeded. - rc_ids = set([_RC_CACHE.id_from_string(a.resource_class) - for a in allocs]) - provider_uuids = set([a.resource_provider.uuid for a in allocs]) - provider_ids = set([a.resource_provider.id for a in allocs]) - usage = sa.select([_ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id, - sql.func.sum(_ALLOC_TBL.c.used).label('used')]) - usage = usage.where( - sa.and_(_ALLOC_TBL.c.resource_class_id.in_(rc_ids), - _ALLOC_TBL.c.resource_provider_id.in_(provider_ids))) - usage = usage.group_by(_ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id) - usage = sa.alias(usage, name='usage') - - inv_join = sql.join(_RP_TBL, _INV_TBL, - sql.and_(_RP_TBL.c.id == _INV_TBL.c.resource_provider_id, - _INV_TBL.c.resource_class_id.in_(rc_ids))) - primary_join = sql.outerjoin(inv_join, usage, - sql.and_( - _INV_TBL.c.resource_provider_id == usage.c.resource_provider_id, - _INV_TBL.c.resource_class_id == usage.c.resource_class_id) - ) - cols_in_output = [ - _RP_TBL.c.id.label('resource_provider_id'), - _RP_TBL.c.uuid, - _RP_TBL.c.generation, - _INV_TBL.c.resource_class_id, - _INV_TBL.c.total, - _INV_TBL.c.reserved, - _INV_TBL.c.allocation_ratio, - _INV_TBL.c.min_unit, - _INV_TBL.c.max_unit, - _INV_TBL.c.step_size, - usage.c.used, - ] - - sel = sa.select(cols_in_output).select_from(primary_join) - sel = sel.where( - sa.and_(_RP_TBL.c.id.in_(provider_ids), - _INV_TBL.c.resource_class_id.in_(rc_ids))) - records = ctx.session.execute(sel) - # Create a map keyed by (rp_uuid, res_class) for the records in the DB - usage_map = {} - provs_with_inv = set() - for record in records: - map_key = (record['uuid'], record['resource_class_id']) - if map_key in usage_map: - raise KeyError("%s already in usage_map, bad query" % str(map_key)) - usage_map[map_key] = record - provs_with_inv.add(record["uuid"]) - # Ensure that all providers have existing inventory - missing_provs = provider_uuids - provs_with_inv - if missing_provs: - class_str = ', '.join([_RC_CACHE.string_from_id(rc_id) - for rc_id in rc_ids]) - provider_str = ', '.join(missing_provs) - raise exception.InvalidInventory(resource_class=class_str, - resource_provider=provider_str) - - res_providers = {} - rp_resource_class_sum = collections.defaultdict( - lambda: collections.defaultdict(int)) - for alloc in allocs: - rc_id = _RC_CACHE.id_from_string(alloc.resource_class) - rp_uuid = alloc.resource_provider.uuid - if rp_uuid not in res_providers: - res_providers[rp_uuid] = alloc.resource_provider - amount_needed = alloc.used - rp_resource_class_sum[rp_uuid][rc_id] += amount_needed - # No use checking usage if we're not asking for anything - if amount_needed == 0: - continue - key = (rp_uuid, rc_id) - try: - usage = usage_map[key] - except KeyError: - # The resource class at rc_id is not in the usage map. - raise exception.InvalidInventory( - resource_class=alloc.resource_class, - resource_provider=rp_uuid) - allocation_ratio = usage['allocation_ratio'] - min_unit = usage['min_unit'] - max_unit = usage['max_unit'] - step_size = usage['step_size'] - - # check min_unit, max_unit, step_size - if (amount_needed < min_unit or amount_needed > max_unit or - amount_needed % step_size != 0): - LOG.warning( - "Allocation for %(rc)s on resource provider %(rp)s " - "violates min_unit, max_unit, or step_size. " - "Requested: %(requested)s, min_unit: %(min_unit)s, " - "max_unit: %(max_unit)s, step_size: %(step_size)s", - {'rc': alloc.resource_class, - 'rp': rp_uuid, - 'requested': amount_needed, - 'min_unit': min_unit, - 'max_unit': max_unit, - 'step_size': step_size}) - raise exception.InvalidAllocationConstraintsViolated( - resource_class=alloc.resource_class, - resource_provider=rp_uuid) - - # usage["used"] can be returned as None - used = usage['used'] or 0 - capacity = (usage['total'] - usage['reserved']) * allocation_ratio - if (capacity < (used + amount_needed) or - capacity < (used + rp_resource_class_sum[rp_uuid][rc_id])): - LOG.warning( - "Over capacity for %(rc)s on resource provider %(rp)s. " - "Needed: %(needed)s, Used: %(used)s, Capacity: %(cap)s", - {'rc': alloc.resource_class, - 'rp': rp_uuid, - 'needed': amount_needed, - 'used': used, - 'cap': capacity}) - raise exception.InvalidAllocationCapacityExceeded( - resource_class=alloc.resource_class, - resource_provider=rp_uuid) - return res_providers - - -@db_api.placement_context_manager.reader -def _get_allocations_by_provider_id(ctx, rp_id): - allocs = sa.alias(_ALLOC_TBL, name="a") - consumers = sa.alias(_CONSUMER_TBL, name="c") - projects = sa.alias(_PROJECT_TBL, name="p") - users = sa.alias(_USER_TBL, name="u") - cols = [ - allocs.c.id, - allocs.c.resource_class_id, - allocs.c.used, - allocs.c.updated_at, - allocs.c.created_at, - consumers.c.id.label("consumer_id"), - consumers.c.generation.label("consumer_generation"), - sql.func.coalesce( - consumers.c.uuid, allocs.c.consumer_id).label("consumer_uuid"), - projects.c.id.label("project_id"), - projects.c.external_id.label("project_external_id"), - users.c.id.label("user_id"), - users.c.external_id.label("user_external_id"), - ] - # TODO(jaypipes): change this join to be on ID not UUID - consumers_join = sa.join( - allocs, consumers, allocs.c.consumer_id == consumers.c.uuid) - projects_join = sa.join( - consumers_join, projects, consumers.c.project_id == projects.c.id) - users_join = sa.join( - projects_join, users, consumers.c.user_id == users.c.id) - sel = sa.select(cols).select_from(users_join) - sel = sel.where(allocs.c.resource_provider_id == rp_id) - - return [dict(r) for r in ctx.session.execute(sel)] - - -@db_api.placement_context_manager.reader -def _get_allocations_by_consumer_uuid(ctx, consumer_uuid): - allocs = sa.alias(_ALLOC_TBL, name="a") - rp = sa.alias(_RP_TBL, name="rp") - consumer = sa.alias(_CONSUMER_TBL, name="c") - project = sa.alias(_PROJECT_TBL, name="p") - user = sa.alias(_USER_TBL, name="u") - cols = [ - allocs.c.id, - allocs.c.resource_provider_id, - rp.c.name.label("resource_provider_name"), - rp.c.uuid.label("resource_provider_uuid"), - rp.c.generation.label("resource_provider_generation"), - allocs.c.resource_class_id, - allocs.c.used, - consumer.c.id.label("consumer_id"), - consumer.c.generation.label("consumer_generation"), - sql.func.coalesce( - consumer.c.uuid, allocs.c.consumer_id).label("consumer_uuid"), - project.c.id.label("project_id"), - project.c.external_id.label("project_external_id"), - user.c.id.label("user_id"), - user.c.external_id.label("user_external_id"), - ] - # Build up the joins of the five tables we need to interact with. - rp_join = sa.join(allocs, rp, allocs.c.resource_provider_id == rp.c.id) - consumer_join = sa.join(rp_join, consumer, - allocs.c.consumer_id == consumer.c.uuid) - project_join = sa.join(consumer_join, project, - consumer.c.project_id == project.c.id) - user_join = sa.join(project_join, user, - consumer.c.user_id == user.c.id) - - sel = sa.select(cols).select_from(user_join) - sel = sel.where(allocs.c.consumer_id == consumer_uuid) - - return [dict(r) for r in ctx.session.execute(sel)] - - -@db_api.placement_context_manager.writer.independent -def _create_incomplete_consumers_for_provider(ctx, rp_id): - # TODO(jaypipes): Remove in Stein after a blocker migration is added. - """Creates consumer record if consumer relationship between allocations -> - consumers table is missing for any allocation on the supplied provider - internal ID, using the "incomplete consumer" project and user CONF options. - """ - alloc_to_consumer = sa.outerjoin( - _ALLOC_TBL, consumer_obj.CONSUMER_TBL, - _ALLOC_TBL.c.consumer_id == consumer_obj.CONSUMER_TBL.c.uuid) - sel = sa.select([_ALLOC_TBL.c.consumer_id]) - sel = sel.select_from(alloc_to_consumer) - sel = sel.where( - sa.and_( - _ALLOC_TBL.c.resource_provider_id == rp_id, - consumer_obj.CONSUMER_TBL.c.id.is_(None))) - missing = ctx.session.execute(sel).fetchall() - if missing: - # Do a single INSERT for all missing consumer relationships for the - # provider - incomplete_proj_id = project_obj.ensure_incomplete_project(ctx) - incomplete_user_id = user_obj.ensure_incomplete_user(ctx) - - cols = [ - _ALLOC_TBL.c.consumer_id, - incomplete_proj_id, - incomplete_user_id, - ] - sel = sa.select(cols) - sel = sel.select_from(alloc_to_consumer) - sel = sel.where( - sa.and_( - _ALLOC_TBL.c.resource_provider_id == rp_id, - consumer_obj.CONSUMER_TBL.c.id.is_(None))) - # NOTE(mnaser): It is possible to have multiple consumers having many - # allocations to the same resource provider, which would - # make the INSERT FROM SELECT fail due to duplicates. - sel = sel.group_by(_ALLOC_TBL.c.consumer_id) - target_cols = ['uuid', 'project_id', 'user_id'] - ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select( - target_cols, sel) - res = ctx.session.execute(ins_stmt) - if res.rowcount > 0: - LOG.info("Online data migration to fix incomplete consumers " - "for resource provider %s has been run. Migrated %d " - "incomplete consumer records on the fly.", rp_id, - res.rowcount) - - -@db_api.placement_context_manager.writer.independent -def _create_incomplete_consumer(ctx, consumer_id): - # TODO(jaypipes): Remove in Stein after a blocker migration is added. - """Creates consumer record if consumer relationship between allocations -> - consumers table is missing for the supplied consumer UUID, using the - "incomplete consumer" project and user CONF options. - """ - alloc_to_consumer = sa.outerjoin( - _ALLOC_TBL, consumer_obj.CONSUMER_TBL, - _ALLOC_TBL.c.consumer_id == consumer_obj.CONSUMER_TBL.c.uuid) - sel = sa.select([_ALLOC_TBL.c.consumer_id]) - sel = sel.select_from(alloc_to_consumer) - sel = sel.where( - sa.and_( - _ALLOC_TBL.c.consumer_id == consumer_id, - consumer_obj.CONSUMER_TBL.c.id.is_(None))) - missing = ctx.session.execute(sel).fetchall() - if missing: - incomplete_proj_id = project_obj.ensure_incomplete_project(ctx) - incomplete_user_id = user_obj.ensure_incomplete_user(ctx) - - ins_stmt = consumer_obj.CONSUMER_TBL.insert().values( - uuid=consumer_id, project_id=incomplete_proj_id, - user_id=incomplete_user_id) - res = ctx.session.execute(ins_stmt) - if res.rowcount > 0: - LOG.info("Online data migration to fix incomplete consumers " - "for consumer %s has been run. Migrated %d incomplete " - "consumer records on the fly.", consumer_id, res.rowcount) - - -@base.VersionedObjectRegistry.register_if(False) -class AllocationList(base.ObjectListBase, base.VersionedObject): - - # The number of times to retry set_allocations if there has - # been a resource provider (not consumer) generation coflict. - RP_CONFLICT_RETRY_COUNT = 10 - - fields = { - 'objects': fields.ListOfObjectsField('Allocation'), - } - - @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) - @db_api.placement_context_manager.writer - def _set_allocations(self, context, allocs): - """Write a set of allocations. - - We must check that there is capacity for each allocation. - If there is not we roll back the entire set. - - :raises `exception.ResourceClassNotFound` if any resource class in any - allocation in allocs cannot be found in either the standard - classes or the DB. - :raises `exception.InvalidAllocationCapacityExceeded` if any inventory - would be exhausted by the allocation. - :raises `InvalidAllocationConstraintsViolated` if any of the - `step_size`, `min_unit` or `max_unit` constraints in an - inventory will be violated by any one of the allocations. - :raises `ConcurrentUpdateDetected` if a generation for a resource - provider or consumer failed its increment check. - """ - # First delete any existing allocations for any consumers. This - # provides a clean slate for the consumers mentioned in the list of - # allocations being manipulated. - consumer_ids = set(alloc.consumer.uuid for alloc in allocs) - for consumer_id in consumer_ids: - _delete_allocations_for_consumer(context, consumer_id) - - # Before writing any allocation records, we check that the submitted - # allocations do not cause any inventory capacity to be exceeded for - # any resource provider and resource class involved in the allocation - # transaction. _check_capacity_exceeded() raises an exception if any - # inventory capacity is exceeded. If capacity is not exceeeded, the - # function returns a list of ResourceProvider objects containing the - # generation of the resource provider at the time of the check. These - # objects are used at the end of the allocation transaction as a guard - # against concurrent updates. - # - # Don't check capacity when alloc.used is zero. Zero is not a valid - # amount when making an allocation (the minimum consumption of a - # resource is one) but is used in this method to indicate a need for - # removal. Providing 0 is controlled at the HTTP API layer where PUT - # /allocations does not allow empty allocations. When POST /allocations - # is implemented it will for the special case of atomically setting and - # removing different allocations in the same request. - # _check_capacity_exceeded will raise a ResourceClassNotFound # if any - # allocation is using a resource class that does not exist. - visited_consumers = {} - visited_rps = _check_capacity_exceeded(context, allocs) - for alloc in allocs: - if alloc.consumer.id not in visited_consumers: - visited_consumers[alloc.consumer.id] = alloc.consumer - - # If alloc.used is set to zero that is a signal that we don't want - # to (re-)create any allocations for this resource class. - # _delete_current_allocs has already wiped out allocations so just - # continue - if alloc.used == 0: - continue - consumer_id = alloc.consumer.uuid - rp = alloc.resource_provider - rc_id = _RC_CACHE.id_from_string(alloc.resource_class) - ins_stmt = _ALLOC_TBL.insert().values( - resource_provider_id=rp.id, - resource_class_id=rc_id, - consumer_id=consumer_id, - used=alloc.used) - res = context.session.execute(ins_stmt) - alloc.id = res.lastrowid - alloc.obj_reset_changes() - - # Generation checking happens here. If the inventory for this resource - # provider changed out from under us, this will raise a - # ConcurrentUpdateDetected which can be caught by the caller to choose - # to try again. It will also rollback the transaction so that these - # changes always happen atomically. - for rp in visited_rps.values(): - rp.generation = _increment_provider_generation(context, rp) - for consumer in visited_consumers.values(): - consumer.increment_generation() - # If any consumers involved in this transaction ended up having no - # allocations, delete the consumer records. Exclude consumers that had - # *some resource* in the allocation list with a total > 0 since clearly - # those consumers have allocations... - cons_with_allocs = set(a.consumer.uuid for a in allocs if a.used > 0) - all_cons = set(c.uuid for c in visited_consumers.values()) - consumers_to_check = all_cons - cons_with_allocs - consumer_obj.delete_consumers_if_no_allocations( - context, consumers_to_check) - - @classmethod - def get_all_by_resource_provider(cls, context, rp): - _create_incomplete_consumers_for_provider(context, rp.id) - db_allocs = _get_allocations_by_provider_id(context, rp.id) - # Build up a list of Allocation objects, setting the Allocation object - # fields to the same-named database record field we got from - # _get_allocations_by_provider_id(). We already have the - # ResourceProvider object so we just pass that object to the Allocation - # object constructor as-is - objs = [] - for rec in db_allocs: - consumer = consumer_obj.Consumer( - context, id=rec['consumer_id'], - uuid=rec['consumer_uuid'], - generation=rec['consumer_generation'], - project=project_obj.Project( - context, id=rec['project_id'], - external_id=rec['project_external_id']), - user=user_obj.User( - context, id=rec['user_id'], - external_id=rec['user_external_id'])) - objs.append( - Allocation( - context, id=rec['id'], resource_provider=rp, - resource_class=_RC_CACHE.string_from_id( - rec['resource_class_id']), - consumer=consumer, - used=rec['used'])) - alloc_list = cls(context, objects=objs) - return alloc_list - - @classmethod - def get_all_by_consumer_id(cls, context, consumer_id): - _create_incomplete_consumer(context, consumer_id) - db_allocs = _get_allocations_by_consumer_uuid(context, consumer_id) - - if db_allocs: - # Build up the Consumer object (it's the same for all allocations - # since we looked up by consumer ID) - db_first = db_allocs[0] - consumer = consumer_obj.Consumer( - context, id=db_first['consumer_id'], - uuid=db_first['consumer_uuid'], - generation=db_first['consumer_generation'], - project=project_obj.Project( - context, id=db_first['project_id'], - external_id=db_first['project_external_id']), - user=user_obj.User( - context, id=db_first['user_id'], - external_id=db_first['user_external_id'])) - - # Build up a list of Allocation objects, setting the Allocation object - # fields to the same-named database record field we got from - # _get_allocations_by_consumer_id(). - # - # NOTE(jaypipes): Unlike with get_all_by_resource_provider(), we do - # NOT already have the ResourceProvider object so we construct a new - # ResourceProvider object below by looking at the resource provider - # fields returned by _get_allocations_by_consumer_id(). - objs = [ - Allocation( - context, id=rec['id'], - resource_provider=ResourceProvider( - context, - id=rec['resource_provider_id'], - uuid=rec['resource_provider_uuid'], - name=rec['resource_provider_name'], - generation=rec['resource_provider_generation']), - resource_class=_RC_CACHE.string_from_id( - rec['resource_class_id']), - consumer=consumer, - used=rec['used']) - for rec in db_allocs - ] - alloc_list = cls(context, objects=objs) - return alloc_list - - def replace_all(self): - """Replace the supplied allocations. - - :note: This method always deletes all allocations for all consumers - referenced in the list of Allocation objects and then replaces - the consumer's allocations with the Allocation objects. In doing - so, it will end up setting the Allocation.id attribute of each - Allocation object. - """ - # Retry _set_allocations server side if there is a - # ResourceProviderConcurrentUpdateDetected. We don't care about - # sleeping, we simply want to reset the resource provider objects - # and try again. For sake of simplicity (and because we don't have - # easy access to the information) we reload all the resource - # providers that may be present. - retries = self.RP_CONFLICT_RETRY_COUNT - while retries: - retries -= 1 - try: - self._set_allocations(self._context, self.objects) - break - except exception.ResourceProviderConcurrentUpdateDetected: - LOG.debug('Retrying allocations write on resource provider ' - 'generation conflict') - # We only want to reload each unique resource provider once. - alloc_rp_uuids = set( - alloc.resource_provider.uuid for alloc in self.objects) - seen_rps = {} - for rp_uuid in alloc_rp_uuids: - seen_rps[rp_uuid] = ResourceProvider.get_by_uuid( - self._context, rp_uuid) - for alloc in self.objects: - rp_uuid = alloc.resource_provider.uuid - alloc.resource_provider = seen_rps[rp_uuid] - else: - # We ran out of retries so we need to raise again. - # The log will automatically have request id info associated with - # it that will allow tracing back to specific allocations. - # Attempting to extract specific consumer or resource provider - # information from the allocations is not coherent as this - # could be multiple consumers and providers. - LOG.warning('Exceeded retry limit of %d on allocations write', - self.RP_CONFLICT_RETRY_COUNT) - raise exception.ResourceProviderConcurrentUpdateDetected() - - def delete_all(self): - consumer_uuids = set(alloc.consumer.uuid for alloc in self.objects) - alloc_ids = [alloc.id for alloc in self.objects] - _delete_allocations_by_ids(self._context, alloc_ids) - consumer_obj.delete_consumers_if_no_allocations( - self._context, consumer_uuids) - - def __repr__(self): - strings = [repr(x) for x in self.objects] - return "AllocationList[" + ", ".join(strings) + "]" - - -@base.VersionedObjectRegistry.register_if(False) -class Usage(base.VersionedObject): - - fields = { - 'resource_class': rc_fields.ResourceClassField(read_only=True), - 'usage': fields.NonNegativeIntegerField(), - } - - @staticmethod - def _from_db_object(context, target, source): - for field in target.fields: - if field not in ('resource_class'): - setattr(target, field, source[field]) - - if 'resource_class' not in target: - rc_str = _RC_CACHE.string_from_id(source['resource_class_id']) - target.resource_class = rc_str - - target._context = context - target.obj_reset_changes() - return target - - -@base.VersionedObjectRegistry.register_if(False) -class UsageList(base.ObjectListBase, base.VersionedObject): - - fields = { - 'objects': fields.ListOfObjectsField('Usage'), - } - - @staticmethod - @db_api.placement_context_manager.reader - def _get_all_by_resource_provider_uuid(context, rp_uuid): - query = (context.session.query(models.Inventory.resource_class_id, - func.coalesce(func.sum(models.Allocation.used), 0)) - .join(models.ResourceProvider, - models.Inventory.resource_provider_id == - models.ResourceProvider.id) - .outerjoin(models.Allocation, - sql.and_(models.Inventory.resource_provider_id == - models.Allocation.resource_provider_id, - models.Inventory.resource_class_id == - models.Allocation.resource_class_id)) - .filter(models.ResourceProvider.uuid == rp_uuid) - .group_by(models.Inventory.resource_class_id)) - result = [dict(resource_class_id=item[0], usage=item[1]) - for item in query.all()] - return result - - @staticmethod - @db_api.placement_context_manager.reader - def _get_all_by_project_user(context, project_id, user_id=None): - query = (context.session.query(models.Allocation.resource_class_id, - func.coalesce(func.sum(models.Allocation.used), 0)) - .join(models.Consumer, - models.Allocation.consumer_id == models.Consumer.uuid) - .join(models.Project, - models.Consumer.project_id == models.Project.id) - .filter(models.Project.external_id == project_id)) - if user_id: - query = query.join(models.User, - models.Consumer.user_id == models.User.id) - query = query.filter(models.User.external_id == user_id) - query = query.group_by(models.Allocation.resource_class_id) - result = [dict(resource_class_id=item[0], usage=item[1]) - for item in query.all()] - return result - - @classmethod - def get_all_by_resource_provider_uuid(cls, context, rp_uuid): - usage_list = cls._get_all_by_resource_provider_uuid(context, rp_uuid) - return base.obj_make_list(context, cls(context), Usage, usage_list) - - @classmethod - def get_all_by_project_user(cls, context, project_id, user_id=None): - usage_list = cls._get_all_by_project_user(context, project_id, - user_id=user_id) - return base.obj_make_list(context, cls(context), Usage, usage_list) - - def __repr__(self): - strings = [repr(x) for x in self.objects] - return "UsageList[" + ", ".join(strings) + "]" - - -@base.VersionedObjectRegistry.register_if(False) -class ResourceClass(base.VersionedObject, base.TimestampedObject): - - MIN_CUSTOM_RESOURCE_CLASS_ID = 10000 - """Any user-defined resource classes must have an identifier greater than - or equal to this number. - """ - - # Retry count for handling possible race condition in creating resource - # class. We don't ever want to hit this, as it is simply a race when - # creating these classes, but this is just a stopgap to prevent a potential - # infinite loop. - RESOURCE_CREATE_RETRY_COUNT = 100 - - fields = { - 'id': fields.IntegerField(read_only=True), - 'name': rc_fields.ResourceClassField(nullable=False), - } - - @staticmethod - def _from_db_object(context, target, source): - for field in target.fields: - setattr(target, field, source[field]) - - target._context = context - target.obj_reset_changes() - return target - - @classmethod - def get_by_name(cls, context, name): - """Return a ResourceClass object with the given string name. - - :param name: String name of the resource class to find - - :raises: ResourceClassNotFound if no such resource class was found - """ - rc = _RC_CACHE.all_from_string(name) - obj = cls(context, id=rc['id'], name=rc['name'], - updated_at=rc['updated_at'], created_at=rc['created_at']) - obj.obj_reset_changes() - return obj - - @staticmethod - @db_api.placement_context_manager.reader - def _get_next_id(context): - """Utility method to grab the next resource class identifier to use for - user-defined resource classes. - """ - query = context.session.query(func.max(models.ResourceClass.id)) - max_id = query.one()[0] - if not max_id: - return ResourceClass.MIN_CUSTOM_RESOURCE_CLASS_ID - else: - return max_id + 1 - - def create(self): - if 'id' in self: - raise exception.ObjectActionError(action='create', - reason='already created') - if 'name' not in self: - raise exception.ObjectActionError(action='create', - reason='name is required') - if self.name in rc_fields.ResourceClass.STANDARD: - raise exception.ResourceClassExists(resource_class=self.name) - - if not self.name.startswith(rc_fields.ResourceClass.CUSTOM_NAMESPACE): - raise exception.ObjectActionError( - action='create', - reason='name must start with ' + - rc_fields.ResourceClass.CUSTOM_NAMESPACE) - - updates = self.obj_get_changes() - # There is the possibility of a race when adding resource classes, as - # the ID is generated locally. This loop catches that exception, and - # retries until either it succeeds, or a different exception is - # encountered. - retries = self.RESOURCE_CREATE_RETRY_COUNT - while retries: - retries -= 1 - try: - rc = self._create_in_db(self._context, updates) - self._from_db_object(self._context, self, rc) - break - except db_exc.DBDuplicateEntry as e: - if 'id' in e.columns: - # Race condition for ID creation; try again - continue - # The duplication is on the other unique column, 'name'. So do - # not retry; raise the exception immediately. - raise exception.ResourceClassExists(resource_class=self.name) - else: - # We have no idea how common it will be in practice for the retry - # limit to be exceeded. We set it high in the hope that we never - # hit this point, but added this log message so we know that this - # specific situation occurred. - LOG.warning("Exceeded retry limit on ID generation while " - "creating ResourceClass %(name)s", - {'name': self.name}) - msg = _("creating resource class %s") % self.name - raise exception.MaxDBRetriesExceeded(action=msg) - - @staticmethod - @db_api.placement_context_manager.writer - def _create_in_db(context, updates): - next_id = ResourceClass._get_next_id(context) - rc = models.ResourceClass() - rc.update(updates) - rc.id = next_id - context.session.add(rc) - return rc - - def destroy(self): - if 'id' not in self: - raise exception.ObjectActionError(action='destroy', - reason='ID attribute not found') - # Never delete any standard resource class, since the standard resource - # classes don't even exist in the database table anyway. - if self.id in (rc['id'] for rc in _RC_CACHE.STANDARDS): - raise exception.ResourceClassCannotDeleteStandard( - resource_class=self.name) - - self._destroy(self._context, self.id, self.name) - _RC_CACHE.clear() - - @staticmethod - @db_api.placement_context_manager.writer - def _destroy(context, _id, name): - # Don't delete the resource class if it is referred to in the - # inventories table. - num_inv = context.session.query(models.Inventory).filter( - models.Inventory.resource_class_id == _id).count() - if num_inv: - raise exception.ResourceClassInUse(resource_class=name) - - res = context.session.query(models.ResourceClass).filter( - models.ResourceClass.id == _id).delete() - if not res: - raise exception.NotFound() - - def save(self): - if 'id' not in self: - raise exception.ObjectActionError(action='save', - reason='ID attribute not found') - updates = self.obj_get_changes() - # Never update any standard resource class, since the standard resource - # classes don't even exist in the database table anyway. - if self.id in (rc['id'] for rc in _RC_CACHE.STANDARDS): - raise exception.ResourceClassCannotUpdateStandard( - resource_class=self.name) - self._save(self._context, self.id, self.name, updates) - _RC_CACHE.clear() - - @staticmethod - @db_api.placement_context_manager.writer - def _save(context, id, name, updates): - db_rc = context.session.query(models.ResourceClass).filter_by( - id=id).first() - db_rc.update(updates) - try: - db_rc.save(context.session) - except db_exc.DBDuplicateEntry: - raise exception.ResourceClassExists(resource_class=name) - - -@base.VersionedObjectRegistry.register_if(False) -class ResourceClassList(base.ObjectListBase, base.VersionedObject): - - fields = { - 'objects': fields.ListOfObjectsField('ResourceClass'), - } - - @staticmethod - @db_api.placement_context_manager.reader - def _get_all(context): - customs = list(context.session.query(models.ResourceClass).all()) - return _RC_CACHE.STANDARDS + customs - - @classmethod - def get_all(cls, context): - resource_classes = cls._get_all(context) - return base.obj_make_list(context, cls(context), - ResourceClass, resource_classes) - - def __repr__(self): - strings = [repr(x) for x in self.objects] - return "ResourceClassList[" + ", ".join(strings) + "]" - - -@base.VersionedObjectRegistry.register_if(False) -class Trait(base.VersionedObject, base.TimestampedObject): - - # All the user-defined traits must begin with this prefix. - CUSTOM_NAMESPACE = 'CUSTOM_' - - fields = { - 'id': fields.IntegerField(read_only=True), - 'name': fields.StringField(nullable=False) - } - - @staticmethod - def _from_db_object(context, trait, db_trait): - for key in trait.fields: - setattr(trait, key, db_trait[key]) - trait.obj_reset_changes() - trait._context = context - return trait - - @staticmethod - @db_api.placement_context_manager.writer - def _create_in_db(context, updates): - trait = models.Trait() - trait.update(updates) - context.session.add(trait) - return trait - - def create(self): - if 'id' in self: - raise exception.ObjectActionError(action='create', - reason='already created') - if 'name' not in self: - raise exception.ObjectActionError(action='create', - reason='name is required') - - updates = self.obj_get_changes() - - try: - db_trait = self._create_in_db(self._context, updates) - except db_exc.DBDuplicateEntry: - raise exception.TraitExists(name=self.name) - - self._from_db_object(self._context, self, db_trait) - - @staticmethod - @db_api.placement_context_manager.writer # trait sync can cause a write - def _get_by_name_from_db(context, name): - result = context.session.query(models.Trait).filter_by( - name=name).first() - if not result: - raise exception.TraitNotFound(names=name) - return result - - @classmethod - def get_by_name(cls, context, name): - db_trait = cls._get_by_name_from_db(context, six.text_type(name)) - return cls._from_db_object(context, cls(), db_trait) - - @staticmethod - @db_api.placement_context_manager.writer - def _destroy_in_db(context, _id, name): - num = context.session.query(models.ResourceProviderTrait).filter( - models.ResourceProviderTrait.trait_id == _id).count() - if num: - raise exception.TraitInUse(name=name) - - res = context.session.query(models.Trait).filter_by( - name=name).delete() - if not res: - raise exception.TraitNotFound(names=name) - - def destroy(self): - if 'name' not in self: - raise exception.ObjectActionError(action='destroy', - reason='name is required') - - if not self.name.startswith(self.CUSTOM_NAMESPACE): - raise exception.TraitCannotDeleteStandard(name=self.name) - - if 'id' not in self: - raise exception.ObjectActionError(action='destroy', - reason='ID attribute not found') - - self._destroy_in_db(self._context, self.id, self.name) - - -@base.VersionedObjectRegistry.register_if(False) -class TraitList(base.ObjectListBase, base.VersionedObject): - - fields = { - 'objects': fields.ListOfObjectsField('Trait') - } - - @staticmethod - @db_api.placement_context_manager.writer # trait sync can cause a write - def _get_all_from_db(context, filters): - if not filters: - filters = {} - - query = context.session.query(models.Trait) - if 'name_in' in filters: - query = query.filter(models.Trait.name.in_( - [six.text_type(n) for n in filters['name_in']] - )) - if 'prefix' in filters: - query = query.filter( - models.Trait.name.like(six.text_type(filters['prefix'] + '%'))) - if 'associated' in filters: - if filters['associated']: - query = query.join(models.ResourceProviderTrait, - models.Trait.id == models.ResourceProviderTrait.trait_id - ).distinct() - else: - query = query.outerjoin(models.ResourceProviderTrait, - models.Trait.id == models.ResourceProviderTrait.trait_id - ).filter(models.ResourceProviderTrait.trait_id == null()) - - return query.all() - - @base.remotable_classmethod - def get_all(cls, context, filters=None): - db_traits = cls._get_all_from_db(context, filters) - return base.obj_make_list(context, cls(context), Trait, db_traits) - - @classmethod - def get_all_by_resource_provider(cls, context, rp): - """Returns a TraitList containing Trait objects for any trait - associated with the supplied resource provider. - """ - db_traits = _get_traits_by_provider_id(context, rp.id) - return base.obj_make_list(context, cls(context), Trait, db_traits) - - -@base.VersionedObjectRegistry.register_if(False) -class AllocationRequestResource(base.VersionedObject): - - fields = { - 'resource_provider': fields.ObjectField('ResourceProvider'), - 'resource_class': rc_fields.ResourceClassField(read_only=True), - 'amount': fields.NonNegativeIntegerField(), - } - - -@base.VersionedObjectRegistry.register_if(False) -class AllocationRequest(base.VersionedObject): - - fields = { - # UUID of (the root of the tree including) the non-sharing resource - # provider associated with this AllocationRequest. Internal use only, - # not included when the object is serialized for output. - 'anchor_root_provider_uuid': fields.UUIDField(), - # Whether all AllocationRequestResources in this AllocationRequest are - # required to be satisfied by the same provider (based on the - # corresponding RequestGroup's use_same_provider attribute). Internal - # use only, not included when the object is serialized for output. - 'use_same_provider': fields.BooleanField(), - 'resource_requests': fields.ListOfObjectsField( - 'AllocationRequestResource' - ), - } - - def __repr__(self): - anchor = (self.anchor_root_provider_uuid[-8:] - if 'anchor_root_provider_uuid' in self else '') - usp = self.use_same_provider if 'use_same_provider' in self else '' - repr_str = ('%s(anchor=...%s, same_provider=%s, ' - 'resource_requests=[%s])' % - (self.obj_name(), anchor, usp, - ', '.join([str(arr) for arr in self.resource_requests]))) - if six.PY2: - repr_str = encodeutils.safe_encode(repr_str, incoming='utf-8') - return repr_str - - -@base.VersionedObjectRegistry.register_if(False) -class ProviderSummaryResource(base.VersionedObject): - - fields = { - 'resource_class': rc_fields.ResourceClassField(read_only=True), - 'capacity': fields.NonNegativeIntegerField(), - 'used': fields.NonNegativeIntegerField(), - # Internal use only; not included when the object is serialized for - # output. - 'max_unit': fields.NonNegativeIntegerField(), - } - - -@base.VersionedObjectRegistry.register_if(False) -class ProviderSummary(base.VersionedObject): - - fields = { - 'resource_provider': fields.ObjectField('ResourceProvider'), - 'resources': fields.ListOfObjectsField('ProviderSummaryResource'), - 'traits': fields.ListOfObjectsField('Trait'), - } - - @property - def resource_class_names(self): - """Helper property that returns a set() of resource class string names - that are included in the provider summary. - """ - return set(res.resource_class for res in self.resources) - - -@db_api.placement_context_manager.reader -def _get_usages_by_provider_tree(ctx, root_ids): - """Returns a row iterator of usage records grouped by provider ID - for all resource providers in all trees indicated in the ``root_ids``. - """ - # We build up a SQL expression that looks like this: - # SELECT - # rp.id as resource_provider_id - # , rp.uuid as resource_provider_uuid - # , inv.resource_class_id - # , inv.total - # , inv.reserved - # , inv.allocation_ratio - # , inv.max_unit - # , usage.used - # FROM resource_providers AS rp - # LEFT JOIN inventories AS inv - # ON rp.id = inv.resource_provider_id - # LEFT JOIN ( - # SELECT resource_provider_id, resource_class_id, SUM(used) as used - # FROM allocations - # JOIN resource_providers - # ON allocations.resource_provider_id = resource_providers.id - # AND (resource_providers.root_provider_id IN($root_ids) - # OR resource_providers.id IN($root_ids)) - # GROUP BY resource_provider_id, resource_class_id - # ) - # AS usage - # ON inv.resource_provider_id = usage.resource_provider_id - # AND inv.resource_class_id = usage.resource_class_id - # WHERE (rp.root_provider_id IN ($root_ids) - # OR resource_providers.id IN($root_ids)) - rpt = sa.alias(_RP_TBL, name="rp") - inv = sa.alias(_INV_TBL, name="inv") - # Build our derived table (subquery in the FROM clause) that sums used - # amounts for resource provider and resource class - derived_alloc_to_rp = sa.join( - _ALLOC_TBL, _RP_TBL, - sa.and_(_ALLOC_TBL.c.resource_provider_id == _RP_TBL.c.id, - # TODO(tetsuro): Remove this OR condition when all - # root_provider_id values are NOT NULL - sa.or_(_RP_TBL.c.root_provider_id.in_(root_ids), - _RP_TBL.c.id.in_(root_ids)) - ) - ) - usage = sa.alias( - sa.select([ - _ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id, - sql.func.sum(_ALLOC_TBL.c.used).label('used'), - ]).select_from(derived_alloc_to_rp).group_by( - _ALLOC_TBL.c.resource_provider_id, - _ALLOC_TBL.c.resource_class_id - ), - name='usage') - # Build a join between the resource providers and inventories table - rpt_inv_join = sa.outerjoin(rpt, inv, - rpt.c.id == inv.c.resource_provider_id) - # And then join to the derived table of usages - usage_join = sa.outerjoin( - rpt_inv_join, - usage, - sa.and_( - usage.c.resource_provider_id == inv.c.resource_provider_id, - usage.c.resource_class_id == inv.c.resource_class_id, - ), - ) - query = sa.select([ - rpt.c.id.label("resource_provider_id"), - rpt.c.uuid.label("resource_provider_uuid"), - inv.c.resource_class_id, - inv.c.total, - inv.c.reserved, - inv.c.allocation_ratio, - inv.c.max_unit, - usage.c.used, - ]).select_from(usage_join).where( - # TODO(tetsuro): Remove this or condition when all - # root_provider_id values are NOT NULL - sa.or_( - rpt.c.root_provider_id.in_(root_ids), - rpt.c.id.in_(root_ids) - ) - ) - return ctx.session.execute(query).fetchall() - - -@db_api.placement_context_manager.reader -def _get_provider_ids_having_any_trait(ctx, traits): - """Returns a set of resource provider internal IDs that have ANY of the - supplied traits. - - :param ctx: Session context to use - :param traits: A map, keyed by trait string name, of trait internal IDs, at - least one of which each provider must have associated with - it. - :raise ValueError: If traits is empty or None. - """ - if not traits: - raise ValueError(_('traits must not be empty')) - - rptt = sa.alias(_RP_TRAIT_TBL, name="rpt") - sel = sa.select([rptt.c.resource_provider_id]) - sel = sel.where(rptt.c.trait_id.in_(traits.values())) - sel = sel.group_by(rptt.c.resource_provider_id) - return set(r[0] for r in ctx.session.execute(sel)) - - -@db_api.placement_context_manager.reader -def _get_provider_ids_having_all_traits(ctx, required_traits): - """Returns a set of resource provider internal IDs that have ALL of the - required traits. - - NOTE: Don't call this method with no required_traits. - - :param ctx: Session context to use - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each provider must have - associated with it - :raise ValueError: If required_traits is empty or None. - """ - if not required_traits: - raise ValueError(_('required_traits must not be empty')) - - rptt = sa.alias(_RP_TRAIT_TBL, name="rpt") - sel = sa.select([rptt.c.resource_provider_id]) - sel = sel.where(rptt.c.trait_id.in_(required_traits.values())) - sel = sel.group_by(rptt.c.resource_provider_id) - # Only get the resource providers that have ALL the required traits, so we - # need to GROUP BY the resource provider and ensure that the - # COUNT(trait_id) is equal to the number of traits we are requiring - num_traits = len(required_traits) - cond = sa.func.count(rptt.c.trait_id) == num_traits - sel = sel.having(cond) - return set(r[0] for r in ctx.session.execute(sel)) - - -@db_api.placement_context_manager.reader -def _has_provider_trees(ctx): - """Simple method that returns whether provider trees (i.e. nested resource - providers) are in use in the deployment at all. This information is used to - switch code paths when attempting to retrieve allocation candidate - information. The code paths are eminently easier to execute and follow for - non-nested scenarios... - - NOTE(jaypipes): The result of this function can be cached extensively. - """ - sel = sa.select([_RP_TBL.c.id]) - sel = sel.where(_RP_TBL.c.parent_provider_id.isnot(None)) - sel = sel.limit(1) - res = ctx.session.execute(sel).fetchall() - return len(res) > 0 - - -@db_api.placement_context_manager.reader -def _get_provider_ids_matching(ctx, resources, required_traits, - forbidden_traits, member_of=None): - """Returns a list of tuples of (internal provider ID, root provider ID) - that have available inventory to satisfy all the supplied requests for - resources. - - :note: This function is used for scenarios that do NOT involve sharing - providers. - - :param ctx: Session context to use - :param resources: A dict, keyed by resource class ID, of the amount - requested of that resource class. - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each provider must have - associated with it - :param forbidden_traits: A map, keyed by trait string name, of forbidden - trait internal IDs that each provider must not - have associated with it - :param member_of: An optional list of list of aggregate UUIDs. If provided, - the allocation_candidates returned will only be for - resource providers that are members of one or more of the - supplied aggregates of each aggregate UUID list. - """ - # The iteratively filtered set of resource provider internal IDs that match - # all the constraints in the request - filtered_rps = set() - if required_traits: - trait_rps = _get_provider_ids_having_all_traits(ctx, required_traits) - filtered_rps = trait_rps - LOG.debug("found %d providers after applying required traits filter " - "(%s)", - len(filtered_rps), list(required_traits)) - if not filtered_rps: - return [] - - # If 'member_of' has values, do a separate lookup to identify the - # resource providers that meet the member_of constraints. - if member_of: - rps_in_aggs = _provider_ids_matching_aggregates(ctx, member_of) - if filtered_rps: - filtered_rps &= set(rps_in_aggs) - else: - filtered_rps = set(rps_in_aggs) - LOG.debug("found %d providers after applying aggregates filter (%s)", - len(filtered_rps), member_of) - if not filtered_rps: - return [] - - forbidden_rp_ids = set() - if forbidden_traits: - forbidden_rp_ids = _get_provider_ids_having_any_trait( - ctx, forbidden_traits) - if filtered_rps: - filtered_rps -= forbidden_rp_ids - LOG.debug("found %d providers after applying forbidden traits " - "filter (%s)", len(filtered_rps), - list(forbidden_traits)) - if not filtered_rps: - return [] - - # Instead of constructing a giant complex SQL statement that joins multiple - # copies of derived usage tables and inventory tables to each other, we do - # one query for each requested resource class. This allows us to log a - # rough idea of which resource class query returned no results (for - # purposes of rough debugging of a single allocation candidates request) as - # well as reduce the necessary knowledge of SQL in order to understand the - # queries being executed here. - # - # NOTE(jaypipes): The efficiency of this operation may be improved by - # passing the trait_rps and/or forbidden_ip_ids iterables to the - # _get_providers_with_resource() function so that we don't have to process - # as many records inside the loop below to remove providers from the - # eventual results list - provs_with_resource = set() - first = True - for rc_id, amount in resources.items(): - rc_name = _RC_CACHE.string_from_id(rc_id) - provs_with_resource = _get_providers_with_resource(ctx, rc_id, amount) - LOG.debug("found %d providers with available %d %s", - len(provs_with_resource), amount, rc_name) - if not provs_with_resource: - return [] - - rc_rp_ids = set(p[0] for p in provs_with_resource) - # The branching below could be collapsed code-wise, but is in place to - # make the debug logging clearer. - if first: - first = False - if filtered_rps: - filtered_rps &= rc_rp_ids - LOG.debug("found %d providers after applying initial " - "aggregate and trait filters", len(filtered_rps)) - else: - filtered_rps = rc_rp_ids - # The following condition is not necessary for the logic; just - # prevents the message from being logged unnecessarily. - if forbidden_rp_ids: - # Forbidden trait filters only need to be applied - # a) on the first iteration; and - # b) if not already set up before the loop - # ...since any providers in the resulting set are the basis - # for intersections, and providers with forbidden traits - # are already absent from that set after we've filtered - # them once. - filtered_rps -= forbidden_rp_ids - LOG.debug("found %d providers after applying forbidden " - "traits", len(filtered_rps)) - else: - filtered_rps &= rc_rp_ids - LOG.debug("found %d providers after filtering by previous result", - len(filtered_rps)) - - if not filtered_rps: - return [] - - # provs_with_resource will contain a superset of providers with IDs still - # in our filtered_rps set. We return the list of tuples of - # (internal provider ID, root internal provider ID) - return [rpids for rpids in provs_with_resource if rpids[0] in filtered_rps] - - -@db_api.placement_context_manager.reader -def _provider_aggregates(ctx, rp_ids): - """Given a list of resource provider internal IDs, returns a dict, - keyed by those provider IDs, of sets of aggregate ids associated - with that provider. - - :raises: ValueError when rp_ids is empty. - - :param ctx: nova.context.RequestContext object - :param rp_ids: list of resource provider IDs - """ - if not rp_ids: - raise ValueError(_("Expected rp_ids to be a list of resource provider " - "internal IDs, but got an empty list.")) - - rpat = sa.alias(_RP_AGG_TBL, name='rpat') - sel = sa.select([rpat.c.resource_provider_id, - rpat.c.aggregate_id]) - sel = sel.where(rpat.c.resource_provider_id.in_(rp_ids)) - res = collections.defaultdict(set) - for r in ctx.session.execute(sel): - res[r[0]].add(r[1]) - return res - - -@db_api.placement_context_manager.reader -def _get_providers_with_resource(ctx, rc_id, amount): - """Returns a set of tuples of (provider ID, root provider ID) of providers - that satisfy the request for a single resource class. - - :param ctx: Session context to use - :param rc_id: Internal ID of resource class to check inventory for - :param amount: Amount of resource being requested - """ - # SELECT rp.id, rp.root_provider_id - # FROM resource_providers AS rp - # JOIN inventories AS inv - # ON rp.id = inv.resource_provider_id - # AND inv.resource_class_id = $RC_ID - # LEFT JOIN ( - # SELECT - # alloc.resource_provider_id, - # SUM(allocs.used) AS used - # FROM allocations AS alloc - # WHERE allocs.resource_class_id = $RC_ID - # GROUP BY allocs.resource_provider_id - # ) AS usage - # ON inv.resource_provider_id = usage.resource_provider_id - # WHERE - # used + $AMOUNT <= ((total - reserved) * inv.allocation_ratio) - # AND inv.min_unit <= $AMOUNT - # AND inv.max_unit >= $AMOUNT - # AND $AMOUNT % inv.step_size == 0 - rpt = sa.alias(_RP_TBL, name="rp") - inv = sa.alias(_INV_TBL, name="inv") - allocs = sa.alias(_ALLOC_TBL, name="alloc") - usage = sa.select([ - allocs.c.resource_provider_id, - sql.func.sum(allocs.c.used).label('used')]) - usage = usage.where(allocs.c.resource_class_id == rc_id) - usage = usage.group_by(allocs.c.resource_provider_id) - usage = sa.alias(usage, name="usage") - where_conds = [ - sql.func.coalesce(usage.c.used, 0) + amount <= ( - (inv.c.total - inv.c.reserved) * inv.c.allocation_ratio), - inv.c.min_unit <= amount, - inv.c.max_unit >= amount, - amount % inv.c.step_size == 0, - ] - rp_to_inv = sa.join( - rpt, inv, sa.and_( - rpt.c.id == inv.c.resource_provider_id, - inv.c.resource_class_id == rc_id)) - inv_to_usage = sa.outerjoin( - rp_to_inv, usage, - inv.c.resource_provider_id == usage.c.resource_provider_id) - sel = sa.select([rpt.c.id, rpt.c.root_provider_id]) - sel = sel.select_from(inv_to_usage) - sel = sel.where(sa.and_(*where_conds)) - res = ctx.session.execute(sel).fetchall() - res = set((r[0], r[1]) for r in res) - # TODO(tetsuro): Bug#1799892: We could have old providers with no root - # provider set and they haven't undergone a data migration yet, - # so we need to set the root_id explicitly here. We remove - # this and when all root_provider_id values are NOT NULL - ret = [] - for rp_tuple in res: - rp_id = rp_tuple[0] - root_id = rp_id if rp_tuple[1] is None else rp_tuple[1] - ret.append((rp_id, root_id)) - return ret - - -@db_api.placement_context_manager.reader -def _get_trees_with_traits(ctx, rp_ids, required_traits, forbidden_traits): - """Given a list of provider IDs, filter them to return a set of tuples of - (provider ID, root provider ID) of providers which belong to a tree that - can satisfy trait requirements. - - :param ctx: Session context to use - :param rp_ids: a set of resource provider IDs - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each provider TREE must - COLLECTIVELY have associated with it - :param forbidden_traits: A map, keyed by trait string name, of trait - internal IDs that a resource provider must - not have. - """ - # We now want to restrict the returned providers to only those provider - # trees that have all our required traits. - # - # The SQL we want looks like this: - # - # SELECT outer_rp.id, outer_rp.root_provider_id - # FROM resource_providers AS outer_rp - # JOIN ( - # SELECT rp.root_provider_id - # FROM resource_providers AS rp - # # Only if we have required traits... - # INNER JOIN resource_provider_traits AS rptt - # ON rp.id = rptt.resource_provider_id - # AND rptt.trait_id IN ($REQUIRED_TRAIT_IDS) - # # Only if we have forbidden_traits... - # LEFT JOIN resource_provider_traits AS rptt_forbid - # ON rp.id = rptt_forbid.resource_provider_id - # AND rptt_forbid.trait_id IN ($FORBIDDEN_TRAIT_IDS) - # WHERE rp.id IN ($RP_IDS) - # # Only if we have forbidden traits... - # AND rptt_forbid.resource_provider_id IS NULL - # GROUP BY rp.root_provider_id - # # Only if have required traits... - # HAVING COUNT(DISTINCT rptt.trait_id) == $NUM_REQUIRED_TRAITS - # ) AS trees_with_traits - # ON outer_rp.root_provider_id = trees_with_traits.root_provider_id - rpt = sa.alias(_RP_TBL, name="rp") - cond = [rpt.c.id.in_(rp_ids)] - subq = sa.select([rpt.c.root_provider_id]) - subq_join = None - if required_traits: - rptt = sa.alias(_RP_TRAIT_TBL, name="rptt") - rpt_to_rptt = sa.join( - rpt, rptt, sa.and_( - rpt.c.id == rptt.c.resource_provider_id, - rptt.c.trait_id.in_(required_traits.values()))) - subq_join = rpt_to_rptt - # Only get the resource providers that have ALL the required traits, - # so we need to GROUP BY the root provider and ensure that the - # COUNT(trait_id) is equal to the number of traits we are requiring - num_traits = len(required_traits) - having_cond = sa.func.count(sa.distinct(rptt.c.trait_id)) == num_traits - subq = subq.having(having_cond) - - # Tack on an additional LEFT JOIN clause inside the derived table if we've - # got forbidden traits in the mix. - if forbidden_traits: - rptt_forbid = sa.alias(_RP_TRAIT_TBL, name="rptt_forbid") - join_to = rpt - if subq_join is not None: - join_to = subq_join - rpt_to_rptt_forbid = sa.outerjoin( - join_to, rptt_forbid, sa.and_( - rpt.c.id == rptt_forbid.c.resource_provider_id, - rptt_forbid.c.trait_id.in_(forbidden_traits.values()))) - cond.append(rptt_forbid.c.resource_provider_id == sa.null()) - subq_join = rpt_to_rptt_forbid - - subq = subq.select_from(subq_join) - subq = subq.where(sa.and_(*cond)) - subq = subq.group_by(rpt.c.root_provider_id) - trees_with_traits = sa.alias(subq, name="trees_with_traits") - - outer_rps = sa.alias(_RP_TBL, name="outer_rps") - outer_to_subq = sa.join( - outer_rps, trees_with_traits, - outer_rps.c.root_provider_id == trees_with_traits.c.root_provider_id) - sel = sa.select([outer_rps.c.id, outer_rps.c.root_provider_id]) - sel = sel.select_from(outer_to_subq) - res = ctx.session.execute(sel).fetchall() - - return [(rp_id, root_id) for rp_id, root_id in res] - - -@db_api.placement_context_manager.reader -def _get_trees_matching_all(ctx, resources, required_traits, forbidden_traits, - sharing, member_of): - """Returns a list of two-tuples (provider internal ID, root provider - internal ID) for providers that satisfy the request for resources. - - If traits are also required, this function only returns results where the - set of providers within a tree that satisfy the resource request - collectively have all the required traits associated with them. This means - that given the following provider tree: - - cn1 - | - --> pf1 (SRIOV_NET_VF:2) - | - --> pf2 (SRIOV_NET_VF:1, HW_NIC_OFFLOAD_GENEVE) - - If a user requests 1 SRIOV_NET_VF resource and no required traits will - return both pf1 and pf2. However, a request for 2 SRIOV_NET_VF and required - trait of HW_NIC_OFFLOAD_GENEVE will return no results (since pf1 is the - only provider with enough inventory of SRIOV_NET_VF but it does not have - the required HW_NIC_OFFLOAD_GENEVE trait). - - :note: This function is used for scenarios to get results for a - RequestGroup with use_same_provider=False. In this scenario, we are able - to use multiple providers within the same provider tree including sharing - providers to satisfy different resources involved in a single RequestGroup. - - :param ctx: Session context to use - :param resources: A dict, keyed by resource class ID, of the amount - requested of that resource class. - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each provider TREE must - COLLECTIVELY have associated with it - :param forbidden_traits: A map, keyed by trait string name, of trait - internal IDs that a resource provider must - not have. - :param sharing: dict, keyed by resource class ID, of lists of resource - provider IDs that share that resource class and can - contribute to the overall allocation request - :param member_of: An optional list of lists of aggregate UUIDs. If - provided, the allocation_candidates returned will only be - for resource providers that are members of one or more of - the supplied aggregates in each aggregate UUID list. - """ - # We first grab the provider trees that have nodes that meet the request - # for each resource class. Once we have this information, we'll then do a - # followup query to winnow the set of resource providers to only those - # provider *trees* that have all of the required traits. - provs_with_inv = set() - # provs_with_inv is a list of three-tuples with the second element being - # the root provider ID and the third being resource class ID. Get the list - # of root provider IDs and get all trees that collectively have all - # required traits. - trees_with_inv = set() - - for rc_id, amount in resources.items(): - rc_provs_with_inv = _get_providers_with_resource(ctx, rc_id, amount) - if not rc_provs_with_inv: - # If there's no providers that have one of the resource classes, - # then we can short-circuit - return [] - rc_trees = set(p[1] for p in rc_provs_with_inv) - provs_with_inv |= set((p[0], p[1], rc_id) for p in rc_provs_with_inv) - - sharing_providers = sharing.get(rc_id) - if sharing_providers: - # There are sharing providers for this resource class, so we - # should also get combinations of (sharing provider, anchor root) - # in addition to (non-sharing provider, anchor root) we already - # have. - rc_provs_with_inv = _anchors_for_sharing_providers( - ctx, sharing_providers, get_id=True) - rc_provs_with_inv = set( - (p[0], p[1], rc_id) for p in rc_provs_with_inv) - rc_trees |= set(p[1] for p in rc_provs_with_inv) - provs_with_inv |= rc_provs_with_inv - - # Filter trees_with_inv to have only trees with enough inventories - # for this resource class. Here "tree" includes sharing providers - # in its terminology - if trees_with_inv: - trees_with_inv &= rc_trees - else: - trees_with_inv = rc_trees - - if not trees_with_inv: - return [] - - # Select only those tuples where there are providers for all requested - # resource classes (trees_with_inv contains the root provider IDs of those - # trees that contain all our requested resources) - provs_with_inv = set(p for p in provs_with_inv if p[1] in trees_with_inv) - - if not provs_with_inv: - return [] - - # If 'member_of' has values, do a separate lookup to identify the - # resource providers that meet the member_of constraints. - if member_of: - rps_in_aggs = _provider_ids_matching_aggregates(ctx, member_of, - rp_ids=trees_with_inv) - if not rps_in_aggs: - # Short-circuit. The user either asked for a non-existing - # aggregate or there were no resource providers that matched - # the requirements... - return [] - provs_with_inv = set(p for p in provs_with_inv if p[1] in rps_in_aggs) - - if (not required_traits and not forbidden_traits) or ( - any(sharing.values())): - # If there were no traits required, there's no difference in how we - # calculate allocation requests between nested and non-nested - # environments, so just short-circuit and return. Or if sharing - # providers are in play, we check the trait constraints later - # in _alloc_candidates_multiple_providers(), so skip. - return list(provs_with_inv) - - # Return the providers where the providers have the available inventory - # capacity and that set of providers (grouped by their tree) have all - # of the required traits and none of the forbidden traits - rp_ids_with_inv = set(p[0] for p in provs_with_inv) - rp_tuples_with_trait = _get_trees_with_traits( - ctx, rp_ids_with_inv, required_traits, forbidden_traits) - - ret = [rp_tuple for rp_tuple in provs_with_inv if ( - rp_tuple[0], rp_tuple[1]) in rp_tuples_with_trait] - - return ret - - -def _build_provider_summaries(context, usages, prov_traits): - """Given a list of dicts of usage information and a map of providers to - their associated string traits, returns a dict, keyed by resource provider - ID, of ProviderSummary objects. - - :param context: nova.context.RequestContext object - :param usages: A list of dicts with the following format: - - { - 'resource_provider_id': , - 'resource_provider_uuid': , - 'resource_class_id': , - 'total': integer, - 'reserved': integer, - 'allocation_ratio': float, - } - :param prov_traits: A dict, keyed by internal resource provider ID, of - string trait names associated with that provider - """ - # Before we go creating provider summary objects, first grab all the - # provider information (including root, parent and UUID information) for - # all providers involved in our operation - rp_ids = set(usage['resource_provider_id'] for usage in usages) - provider_ids = _provider_ids_from_rp_ids(context, rp_ids) - - # Build up a dict, keyed by internal resource provider ID, of - # ProviderSummary objects containing one or more ProviderSummaryResource - # objects representing the resources the provider has inventory for. - summaries = {} - for usage in usages: - rp_id = usage['resource_provider_id'] - summary = summaries.get(rp_id) - if not summary: - pids = provider_ids[rp_id] - summary = ProviderSummary( - context, - resource_provider=ResourceProvider( - context, id=pids.id, uuid=pids.uuid, - root_provider_uuid=pids.root_uuid, - parent_provider_uuid=pids.parent_uuid), - resources=[], - ) - summaries[rp_id] = summary - - traits = prov_traits[rp_id] - summary.traits = [Trait(context, name=tname) for tname in traits] - - rc_id = usage['resource_class_id'] - if rc_id is None: - # NOTE(tetsuro): This provider doesn't have any inventory itself. - # But we include this provider in summaries since another - # provider in the same tree will be in the "allocation_request". - # Let's skip the following and leave "ProviderSummary.resources" - # field empty. - continue - # NOTE(jaypipes): usage['used'] may be None due to the LEFT JOIN of - # the usages subquery, so we coerce NULL values to 0 here. - used = usage['used'] or 0 - allocation_ratio = usage['allocation_ratio'] - cap = int((usage['total'] - usage['reserved']) * allocation_ratio) - rc_name = _RC_CACHE.string_from_id(rc_id) - rpsr = ProviderSummaryResource( - context, - resource_class=rc_name, - capacity=cap, - used=used, - max_unit=usage['max_unit'], - ) - summary.resources.append(rpsr) - return summaries - - -def _aggregates_associated_with_providers(a, b, prov_aggs): - """quickly check if the two rps are in the same aggregates - - :param a: resource provider ID for first provider - :param b: resource provider ID for second provider - :param prov_aggs: a dict keyed by resource provider IDs, of sets - of aggregate ids associated with that provider - """ - a_aggs = prov_aggs[a] - b_aggs = prov_aggs[b] - return a_aggs & b_aggs - - -def _shared_allocation_request_resources(ctx, ns_rp_id, requested_resources, - sharing, summaries, prov_aggs): - """Returns a dict, keyed by resource class ID, of lists of - AllocationRequestResource objects that represent resources that are - provided by a sharing provider. - - :param ctx: nova.context.RequestContext object - :param ns_rp_id: an internal ID of a non-sharing resource provider - :param requested_resources: dict, keyed by resource class ID, of amounts - being requested for that resource class - :param sharing: dict, keyed by resource class ID, of lists of resource - provider IDs that share that resource class and can - contribute to the overall allocation request - :param summaries: dict, keyed by resource provider ID, of ProviderSummary - objects containing usage and trait information for - resource providers involved in the overall request - :param prov_aggs: dict, keyed by resource provider ID, of sets of - aggregate ids associated with that provider. - """ - res_requests = collections.defaultdict(list) - for rc_id in sharing: - for rp_id in sharing[rc_id]: - aggs_in_both = _aggregates_associated_with_providers( - ns_rp_id, rp_id, prov_aggs) - if not aggs_in_both: - continue - summary = summaries[rp_id] - rp_uuid = summary.resource_provider.uuid - res_req = AllocationRequestResource( - ctx, - resource_provider=ResourceProvider(ctx, uuid=rp_uuid), - resource_class=_RC_CACHE.string_from_id(rc_id), - amount=requested_resources[rc_id], - ) - res_requests[rc_id].append(res_req) - return res_requests - - -def _allocation_request_for_provider(ctx, requested_resources, provider): - """Returns an AllocationRequest object containing AllocationRequestResource - objects for each resource class in the supplied requested resources dict. - - :param ctx: nova.context.RequestContext object - :param requested_resources: dict, keyed by resource class ID, of amounts - being requested for that resource class - :param provider: ResourceProvider object representing the provider of the - resources. - """ - resource_requests = [ - AllocationRequestResource( - ctx, resource_provider=provider, - resource_class=_RC_CACHE.string_from_id(rc_id), - amount=amount, - ) for rc_id, amount in requested_resources.items() - ] - # NOTE(efried): This method only produces an AllocationRequest with its - # anchor in its own tree. If the provider is a sharing provider, the - # caller needs to identify the other anchors with which it might be - # associated. - return AllocationRequest( - ctx, resource_requests=resource_requests, - anchor_root_provider_uuid=provider.root_provider_uuid) - - -def _check_traits_for_alloc_request(res_requests, summaries, prov_traits, - required_traits, forbidden_traits): - """Given a list of AllocationRequestResource objects, check if that - combination can provide trait constraints. If it can, returns all - resource provider internal IDs in play, else return an empty list. - - TODO(tetsuro): For optimization, we should move this logic to SQL in - _get_trees_matching_all(). - - :param res_requests: a list of AllocationRequestResource objects that have - resource providers to be checked if they collectively - satisfy trait constraints in the required_traits and - forbidden_traits parameters. - :param summaries: dict, keyed by resource provider ID, of ProviderSummary - objects containing usage and trait information for - resource providers involved in the overall request - :param prov_traits: A dict, keyed by internal resource provider ID, of - string trait names associated with that provider - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each *allocation request's - set of providers* must *collectively* have - associated with them - :param forbidden_traits: A map, keyed by trait string name, of trait - internal IDs that a resource provider must - not have. - """ - all_prov_ids = [] - all_traits = set() - for res_req in res_requests: - rp_uuid = res_req.resource_provider.uuid - for rp_id, summary in summaries.items(): - if summary.resource_provider.uuid == rp_uuid: - break - rp_traits = set(prov_traits.get(rp_id, [])) - - # Check if there are forbidden_traits - conflict_traits = set(forbidden_traits) & set(rp_traits) - if conflict_traits: - LOG.debug('Excluding resource provider %s, it has ' - 'forbidden traits: (%s).', - rp_id, ', '.join(conflict_traits)) - return [] - - all_prov_ids.append(rp_id) - all_traits |= rp_traits - - # Check if there are missing traits - missing_traits = set(required_traits) - all_traits - if missing_traits: - LOG.debug('Excluding a set of allocation candidate %s : ' - 'missing traits %s are not satisfied.', - all_prov_ids, ','.join(missing_traits)) - return [] - - return all_prov_ids - - -def _alloc_candidates_single_provider(ctx, requested_resources, rp_tuples): - """Returns a tuple of (allocation requests, provider summaries) for a - supplied set of requested resource amounts and resource providers. The - supplied resource providers have capacity to satisfy ALL of the resources - in the requested resources as well as ALL required traits that were - requested by the user. - - This is used in two circumstances: - - To get results for a RequestGroup with use_same_provider=True. - - As an optimization when no sharing providers satisfy any of the requested - resources, and nested providers are not in play. - In these scenarios, we can more efficiently build the list of - AllocationRequest and ProviderSummary objects due to not having to - determine requests across multiple providers. - - :param ctx: nova.context.RequestContext object - :param requested_resources: dict, keyed by resource class ID, of amounts - being requested for that resource class - :param rp_tuples: List of two-tuples of (provider ID, root provider ID)s - for providers that matched the requested resources - """ - if not rp_tuples: - return [], [] - - # Get all root resource provider IDs. - root_ids = set(p[1] for p in rp_tuples) - - # Grab usage summaries for each provider - usages = _get_usages_by_provider_tree(ctx, root_ids) - - # Get a dict, keyed by resource provider internal ID, of trait string names - # that provider has associated with it - prov_traits = _get_traits_by_provider_tree(ctx, root_ids) - - # Get a dict, keyed by resource provider internal ID, of ProviderSummary - # objects for all providers - summaries = _build_provider_summaries(ctx, usages, prov_traits) - - # Next, build up a list of allocation requests. These allocation requests - # are AllocationRequest objects, containing resource provider UUIDs, - # resource class names and amounts to consume from that resource provider - alloc_requests = [] - for rp_id, root_id in rp_tuples: - rp_summary = summaries[rp_id] - req_obj = _allocation_request_for_provider( - ctx, requested_resources, rp_summary.resource_provider) - alloc_requests.append(req_obj) - # If this is a sharing provider, we have to include an extra - # AllocationRequest for every possible anchor. - traits = [trait.name for trait in rp_summary.traits] - if os_traits.MISC_SHARES_VIA_AGGREGATE in traits: - anchors = set([p[1] for p in _anchors_for_sharing_providers( - ctx, [rp_summary.resource_provider.id])]) - for anchor in anchors: - # We already added self - if anchor == rp_summary.resource_provider.root_provider_uuid: - continue - req_obj = copy.deepcopy(req_obj) - req_obj.anchor_root_provider_uuid = anchor - alloc_requests.append(req_obj) - return alloc_requests, list(summaries.values()) - - -def _alloc_candidates_multiple_providers(ctx, requested_resources, - required_traits, forbidden_traits, rp_tuples): - """Returns a tuple of (allocation requests, provider summaries) for a - supplied set of requested resource amounts and tuples of - (rp_id, root_id, rc_id). The supplied resource provider trees have - capacity to satisfy ALL of the resources in the requested resources as - well as ALL required traits that were requested by the user. - - This is a code path to get results for a RequestGroup with - use_same_provider=False. In this scenario, we are able to use multiple - providers within the same provider tree including sharing providers to - satisfy different resources involved in a single request group. - - :param ctx: nova.context.RequestContext object - :param requested_resources: dict, keyed by resource class ID, of amounts - being requested for that resource class - :param required_traits: A map, keyed by trait string name, of required - trait internal IDs that each *allocation request's - set of providers* must *collectively* have - associated with them - :param forbidden_traits: A map, keyed by trait string name, of trait - internal IDs that a resource provider must - not have. - :param rp_tuples: List of tuples of (provider ID, anchor root provider ID, - resource class ID)s for providers that matched the - requested resources - """ - if not rp_tuples: - return [], [] - - # Get all the root resource provider IDs. We should include the first - # values of rp_tuples because while sharing providers are root providers, - # they have their "anchor" providers for the second value. - root_ids = set(p[0] for p in rp_tuples) | set(p[1] for p in rp_tuples) - - # Grab usage summaries for each provider in the trees - usages = _get_usages_by_provider_tree(ctx, root_ids) - - # Get a dict, keyed by resource provider internal ID, of trait string names - # that provider has associated with it - prov_traits = _get_traits_by_provider_tree(ctx, root_ids) - - # Get a dict, keyed by resource provider internal ID, of ProviderSummary - # objects for all providers - summaries = _build_provider_summaries(ctx, usages, prov_traits) - - # Get a dict, keyed by root provider internal ID, of a dict, keyed by - # resource class internal ID, of lists of AllocationRequestResource objects - tree_dict = collections.defaultdict(lambda: collections.defaultdict(list)) - - for rp_id, root_id, rc_id in rp_tuples: - rp_summary = summaries[rp_id] - tree_dict[root_id][rc_id].append( - AllocationRequestResource( - ctx, resource_provider=rp_summary.resource_provider, - resource_class=_RC_CACHE.string_from_id(rc_id), - amount=requested_resources[rc_id])) - - # Next, build up a list of allocation requests. These allocation requests - # are AllocationRequest objects, containing resource provider UUIDs, - # resource class names and amounts to consume from that resource provider - alloc_requests = [] - - # Build a list of lists of provider internal IDs that end up in - # allocation request objects. This is used to ensure we don't end up - # having allocation requests with duplicate sets of resource providers. - alloc_prov_ids = [] - - # Let's look into each tree - for root_id, alloc_dict in tree_dict.items(): - # Get request_groups, which is a list of lists of - # AllocationRequestResource(ARR) per requested resource class(rc). - # For example, if we have the alloc_dict: - # {rc1_id: [ARR(rc1, rp1), ARR(rc1, rp2)], - # rc2_id: [ARR(rc2, rp1), ARR(rc2, rp2)], - # rc3_id: [ARR(rc3, rp1)]} - # then the request_groups would be something like - # [[ARR(rc1, rp1), ARR(rc1, rp2)], - # [ARR(rc2, rp1), ARR(rc2, rp2)], - # [ARR(rc3, rp1)]] - # , which should be ordered by the resource class id. - request_groups = [val for key, val in sorted(alloc_dict.items())] - - root_summary = summaries[root_id] - root_uuid = root_summary.resource_provider.uuid - - # Using itertools.product, we get all the combinations of resource - # providers in a tree. - # For example, the sample in the comment above becomes: - # [(ARR(rc1, ss1), ARR(rc2, ss1), ARR(rc3, ss1)), - # (ARR(rc1, ss1), ARR(rc2, ss2), ARR(rc3, ss1)), - # (ARR(rc1, ss2), ARR(rc2, ss1), ARR(rc3, ss1)), - # (ARR(rc1, ss2), ARR(rc2, ss2), ARR(rc3, ss1))] - for res_requests in itertools.product(*request_groups): - all_prov_ids = _check_traits_for_alloc_request(res_requests, - summaries, prov_traits, required_traits, forbidden_traits) - if (not all_prov_ids) or (all_prov_ids in alloc_prov_ids): - # This combination doesn't satisfy trait constraints, - # ...or we already have this permutation, which happens - # when multiple sharing providers with different resource - # classes are in one request. - continue - alloc_prov_ids.append(all_prov_ids) - alloc_requests.append( - AllocationRequest(ctx, resource_requests=list(res_requests), - anchor_root_provider_uuid=root_uuid) - ) - return alloc_requests, list(summaries.values()) - - -@db_api.placement_context_manager.reader -def _get_traits_by_provider_tree(ctx, root_ids): - """Returns a dict, keyed by provider IDs for all resource providers - in all trees indicated in the ``root_ids``, of string trait names - associated with that provider. - - :raises: ValueError when root_ids is empty. - - :param ctx: nova.context.RequestContext object - :param root_ids: list of root resource provider IDs - """ - if not root_ids: - raise ValueError(_("Expected root_ids to be a list of root resource " - "provider internal IDs, but got an empty list.")) - - rpt = sa.alias(_RP_TBL, name='rpt') - rptt = sa.alias(_RP_TRAIT_TBL, name='rptt') - tt = sa.alias(_TRAIT_TBL, name='t') - rpt_rptt = sa.join(rpt, rptt, rpt.c.id == rptt.c.resource_provider_id) - j = sa.join(rpt_rptt, tt, rptt.c.trait_id == tt.c.id) - sel = sa.select([rptt.c.resource_provider_id, tt.c.name]).select_from(j) - sel = sel.where(rpt.c.root_provider_id.in_(root_ids)) - res = collections.defaultdict(list) - for r in ctx.session.execute(sel): - res[r[0]].append(r[1]) - return res - - -@db_api.placement_context_manager.reader -def _trait_ids_from_names(ctx, names): - """Given a list of string trait names, returns a dict, keyed by those - string names, of the corresponding internal integer trait ID. - - :raises: ValueError when names is empty. - - :param ctx: nova.context.RequestContext object - :param names: list of string trait names - """ - if not names: - raise ValueError(_("Expected names to be a list of string trait " - "names, but got an empty list.")) - - # Avoid SAWarnings about unicode types... - unames = map(six.text_type, names) - tt = sa.alias(_TRAIT_TBL, name='t') - sel = sa.select([tt.c.name, tt.c.id]).where(tt.c.name.in_(unames)) - return {r[0]: r[1] for r in ctx.session.execute(sel)} - - -def _rp_rc_key(rp, rc): - """Creates hashable key unique to a provider + resource class.""" - return rp.uuid, rc - - -def _consolidate_allocation_requests(areqs): - """Consolidates a list of AllocationRequest into one. - - :param areqs: A list containing one AllocationRequest for each input - RequestGroup. This may mean that multiple resource_requests - contain resource amounts of the same class from the same provider. - :return: A single consolidated AllocationRequest, containing no - resource_requests with duplicated (resource_provider, - resource_class). - """ - # Construct a dict, keyed by resource provider UUID + resource class, of - # AllocationRequestResource, consolidating as we go. - arrs_by_rp_rc = {} - # areqs must have at least one element. Save the anchor to populate the - # returned AllocationRequest. - anchor_rp_uuid = areqs[0].anchor_root_provider_uuid - for areq in areqs: - # Sanity check: the anchor should be the same for every areq - if anchor_rp_uuid != areq.anchor_root_provider_uuid: - # This should never happen. If it does, it's a dev bug. - raise ValueError( - _("Expected every AllocationRequest in " - "`_consolidate_allocation_requests` to have the same " - "anchor!")) - for arr in areq.resource_requests: - key = _rp_rc_key(arr.resource_provider, arr.resource_class) - if key not in arrs_by_rp_rc: - arrs_by_rp_rc[key] = copy.deepcopy(arr) - else: - arrs_by_rp_rc[key].amount += arr.amount - return AllocationRequest( - resource_requests=list(arrs_by_rp_rc.values()), - anchor_root_provider_uuid=anchor_rp_uuid) - - -def _satisfies_group_policy(areqs, group_policy, num_granular_groups): - """Applies group_policy to a list of AllocationRequest. - - Returns True or False, indicating whether this list of - AllocationRequest satisfies group_policy, as follows: - - * "isolate": Each AllocationRequest with use_same_provider=True - is satisfied by a single resource provider. If the "isolate" - policy is in effect, each such AllocationRequest must be - satisfied by a *unique* resource provider. - * "none" or None: Always returns True. - - :param areqs: A list containing one AllocationRequest for each input - RequestGroup. - :param group_policy: String indicating how RequestGroups should interact - with each other. If the value is "isolate", we will return False - if AllocationRequests that came from RequestGroups keyed by - nonempty suffixes are satisfied by the same provider. - :param num_granular_groups: The number of granular (use_same_provider=True) - RequestGroups in the request. - :return: True if areqs satisfies group_policy; False otherwise. - """ - if group_policy != 'isolate': - # group_policy="none" means no filtering - return True - - # The number of unique resource providers referenced in the request groups - # having use_same_provider=True must be equal to the number of granular - # groups. - num_granular_groups_in_areqs = len(set( - # We can reliably use the first resource_request's provider: all the - # resource_requests are satisfied by the same provider by definition - # because use_same_provider is True. - areq.resource_requests[0].resource_provider.uuid - for areq in areqs - if areq.use_same_provider)) - if num_granular_groups == num_granular_groups_in_areqs: - return True - LOG.debug('Excluding the following set of AllocationRequest because ' - 'group_policy=isolate and the number of granular groups in the ' - 'set (%d) does not match the number of granular groups in the ' - 'request (%d): %s', - num_granular_groups_in_areqs, num_granular_groups, str(areqs)) - return False - - -def _exceeds_capacity(areq, psum_res_by_rp_rc): - """Checks a (consolidated) AllocationRequest against the provider summaries - to ensure that it does not exceed capacity. - - Exceeding capacity can mean the total amount (already used plus this - allocation) exceeds the total inventory amount; or this allocation exceeds - the max_unit in the inventory record. - - :param areq: An AllocationRequest produced by the - `_consolidate_allocation_requests` method. - :param psum_res_by_rp_rc: A dict, keyed by provider + resource class via - _rp_rc_key, of ProviderSummaryResource. - :return: True if areq exceeds capacity; False otherwise. - """ - for arr in areq.resource_requests: - key = _rp_rc_key(arr.resource_provider, arr.resource_class) - psum_res = psum_res_by_rp_rc[key] - if psum_res.used + arr.amount > psum_res.capacity: - LOG.debug('Excluding the following AllocationRequest because used ' - '(%d) + amount (%d) > capacity (%d) for resource class ' - '%s: %s', - psum_res.used, arr.amount, psum_res.capacity, - arr.resource_class, str(areq)) - return True - if arr.amount > psum_res.max_unit: - LOG.debug('Excluding the following AllocationRequest because ' - 'amount (%d) > max_unit (%d) for resource class %s: %s', - arr.amount, psum_res.max_unit, arr.resource_class, - str(areq)) - return True - return False - - -def _merge_candidates(candidates, group_policy=None): - """Given a dict, keyed by RequestGroup suffix, of tuples of - (allocation_requests, provider_summaries), produce a single tuple of - (allocation_requests, provider_summaries) that appropriately incorporates - the elements from each. - - Each (alloc_reqs, prov_sums) in `candidates` satisfies one RequestGroup. - This method creates a list of alloc_reqs, *each* of which satisfies *all* - of the RequestGroups. - - For that merged list of alloc_reqs, a corresponding provider_summaries is - produced. - - :param candidates: A dict, keyed by integer suffix or '', of tuples of - (allocation_requests, provider_summaries) to be merged. - :param group_policy: String indicating how RequestGroups should interact - with each other. If the value is "isolate", we will filter out - candidates where AllocationRequests that came from RequestGroups - keyed by nonempty suffixes are satisfied by the same provider. - :return: A tuple of (allocation_requests, provider_summaries). - """ - # Build a dict, keyed by anchor root provider UUID, of dicts, keyed by - # suffix, of nonempty lists of AllocationRequest. Each inner dict must - # possess all of the suffix keys to be viable (i.e. contains at least - # one AllocationRequest per RequestGroup). - # - # areq_lists_by_anchor = - # { anchor_root_provider_uuid: { - # '': [AllocationRequest, ...], \ This dict must contain - # '1': [AllocationRequest, ...], \ exactly one nonempty list per - # ... / suffix to be viable. That - # '42': [AllocationRequest, ...], / filtering is done later. - # }, - # ... - # } - areq_lists_by_anchor = collections.defaultdict( - lambda: collections.defaultdict(list)) - # Save off all the provider summaries lists - we'll use 'em later. - all_psums = [] - # Construct a dict, keyed by resource provider + resource class, of - # ProviderSummaryResource. This will be used to do a final capacity - # check/filter on each merged AllocationRequest. - psum_res_by_rp_rc = {} - for suffix, (areqs, psums) in candidates.items(): - for areq in areqs: - anchor = areq.anchor_root_provider_uuid - areq_lists_by_anchor[anchor][suffix].append(areq) - for psum in psums: - all_psums.append(psum) - for psum_res in psum.resources: - key = _rp_rc_key( - psum.resource_provider, psum_res.resource_class) - psum_res_by_rp_rc[key] = psum_res - - # Create all combinations picking one AllocationRequest from each list - # for each anchor. - areqs = [] - all_suffixes = set(candidates) - num_granular_groups = len(all_suffixes - set([''])) - for areq_lists_by_suffix in areq_lists_by_anchor.values(): - # Filter out any entries that don't have allocation requests for - # *all* suffixes (i.e. all RequestGroups) - if set(areq_lists_by_suffix) != all_suffixes: - continue - # We're using itertools.product to go from this: - # areq_lists_by_suffix = { - # '': [areq__A, areq__B, ...], - # '1': [areq_1_A, areq_1_B, ...], - # ... - # '42': [areq_42_A, areq_42_B, ...], - # } - # to this: - # [ [areq__A, areq_1_A, ..., areq_42_A], Each of these lists is one - # [areq__A, areq_1_A, ..., areq_42_B], areq_list in the loop below. - # [areq__A, areq_1_B, ..., areq_42_A], each areq_list contains one - # [areq__A, areq_1_B, ..., areq_42_B], AllocationRequest from each - # [areq__B, areq_1_A, ..., areq_42_A], RequestGroup. So taken as a - # [areq__B, areq_1_A, ..., areq_42_B], whole, each list is a viable - # [areq__B, areq_1_B, ..., areq_42_A], (preliminary) candidate to - # [areq__B, areq_1_B, ..., areq_42_B], return. - # ..., - # ] - for areq_list in itertools.product( - *list(areq_lists_by_suffix.values())): - # At this point, each AllocationRequest in areq_list is still - # marked as use_same_provider. This is necessary to filter by group - # policy, which enforces how these interact with each other. - if not _satisfies_group_policy( - areq_list, group_policy, num_granular_groups): - continue - # Now we go from this (where 'arr' is AllocationRequestResource): - # [ areq__B(arrX, arrY, arrZ), - # areq_1_A(arrM, arrN), - # ..., - # areq_42_B(arrQ) - # ] - # to this: - # areq_combined(arrX, arrY, arrZ, arrM, arrN, arrQ) - # Note that this discards the information telling us which - # RequestGroup led to which piece of the final AllocationRequest. - # We needed that to be present for the previous filter; we need it - # to be *absent* for the next one (and for the final output). - areq = _consolidate_allocation_requests(areq_list) - # Since we sourced this AllocationRequest from multiple - # *independent* queries, it's possible that the combined result - # now exceeds capacity where amounts of the same RP+RC were - # folded together. So do a final capacity check/filter. - if _exceeds_capacity(areq, psum_res_by_rp_rc): - continue - areqs.append(areq) - - # It's possible we've filtered out everything. If so, short out. - if not areqs: - return [], [] - - # Now we have to produce provider summaries. The provider summaries in - # the candidates input contain all the information; we just need to - # filter it down to only the providers in trees represented by our merged - # list of allocation requests. - tree_uuids = set() - for areq in areqs: - for arr in areq.resource_requests: - tree_uuids.add(arr.resource_provider.root_provider_uuid) - psums = [psum for psum in all_psums if - psum.resource_provider.root_provider_uuid in tree_uuids] - - return areqs, psums - - -@base.VersionedObjectRegistry.register_if(False) -class AllocationCandidates(base.VersionedObject): - """The AllocationCandidates object is a collection of possible allocations - that match some request for resources, along with some summary information - about the resource providers involved in these allocation candidates. - """ - - fields = { - # A collection of allocation possibilities that can be attempted by the - # caller that would, at the time of calling, meet the requested - # resource constraints - 'allocation_requests': fields.ListOfObjectsField('AllocationRequest'), - # Information about usage and inventory that relate to any provider - # contained in any of the AllocationRequest objects in the - # allocation_requests field - 'provider_summaries': fields.ListOfObjectsField('ProviderSummary'), - } - - @classmethod - def get_by_requests(cls, context, requests, limit=None, group_policy=None): - """Returns an AllocationCandidates object containing all resource - providers matching a set of supplied resource constraints, with a set - of allocation requests constructed from that list of resource - providers. If CONF.placement.randomize_allocation_candidates is True - (default is False) then the order of the allocation requests will - be randomized. - - :param context: Nova RequestContext. - :param requests: Dict, keyed by suffix, of - nova.api.openstack.placement.util.RequestGroup - :param limit: An integer, N, representing the maximum number of - allocation candidates to return. If - CONF.placement.randomize_allocation_candidates is True - this will be a random sampling of N of the available - results. If False then the first N results, in whatever - order the database picked them, will be returned. In - either case if there are fewer than N total results, - all the results will be returned. - :param group_policy: String indicating how RequestGroups with - use_same_provider=True should interact with each - other. If the value is "isolate", we will filter - out allocation requests where any such - RequestGroups are satisfied by the same RP. - :return: An instance of AllocationCandidates with allocation_requests - and provider_summaries satisfying `requests`, limited - according to `limit`. - """ - alloc_reqs, provider_summaries = cls._get_by_requests( - context, requests, limit=limit, group_policy=group_policy) - return cls( - context, - allocation_requests=alloc_reqs, - provider_summaries=provider_summaries, - ) - - @staticmethod - def _get_by_one_request(context, request, sharing_providers, has_trees): - """Get allocation candidates for one RequestGroup. - - Must be called from within an placement_context_manager.reader - (or writer) context. - - :param context: Nova RequestContext. - :param request: One nova.api.openstack.placement.util.RequestGroup - :param sharing_providers: dict, keyed by resource class internal ID, of - the set of provider IDs containing shared - inventory of that resource class - :param has_trees: bool indicating there is some level of nesting in the - environment (if there isn't, we take faster, simpler - code paths) - :return: A tuple of (allocation_requests, provider_summaries) - satisfying `request`. - """ - # Transform resource string names to internal integer IDs - resources = { - _RC_CACHE.id_from_string(key): value - for key, value in request.resources.items() - } - - # maps the trait name to the trait internal ID - required_trait_map = {} - forbidden_trait_map = {} - for trait_map, traits in ( - (required_trait_map, request.required_traits), - (forbidden_trait_map, request.forbidden_traits)): - if traits: - trait_map.update(_trait_ids_from_names(context, traits)) - # Double-check that we found a trait ID for each requested name - if len(trait_map) != len(traits): - missing = traits - set(trait_map) - raise exception.TraitNotFound(names=', '.join(missing)) - - member_of = request.member_of - - any_sharing = any(sharing_providers.values()) - if not request.use_same_provider and (has_trees or any_sharing): - # TODO(jaypipes): The check/callout to handle trees goes here. - # Build a dict, keyed by resource class internal ID, of lists of - # internal IDs of resource providers that share some inventory for - # each resource class requested. - # If there aren't any providers that have any of the - # required traits, just exit early... - if required_trait_map: - # TODO(cdent): Now that there is also a forbidden_trait_map - # it should be possible to further optimize this attempt at - # a quick return, but we leave that to future patches for - # now. - trait_rps = _get_provider_ids_having_any_trait( - context, required_trait_map) - if not trait_rps: - return [], [] - rp_tuples = _get_trees_matching_all(context, resources, - required_trait_map, forbidden_trait_map, - sharing_providers, member_of) - return _alloc_candidates_multiple_providers(context, resources, - required_trait_map, forbidden_trait_map, rp_tuples) - - # Either we are processing a single-RP request group, or there are no - # sharing providers that (help) satisfy the request. Get a list of - # tuples of (internal provider ID, root provider ID) that have ALL - # the requested resources and more efficiently construct the - # allocation requests. - rp_tuples = _get_provider_ids_matching(context, resources, - required_trait_map, - forbidden_trait_map, member_of) - return _alloc_candidates_single_provider(context, resources, rp_tuples) - - @classmethod - # TODO(efried): This is only a writer context because it accesses the - # resource_providers table via ResourceProvider.get_by_uuid, which does - # data migration to populate the root_provider_uuid. Change this back to a - # reader when that migration is no longer happening. - @db_api.placement_context_manager.writer - def _get_by_requests(cls, context, requests, limit=None, - group_policy=None): - # TODO(jaypipes): Make a RequestGroupContext object and put these - # pieces of information in there, passing the context to the various - # internal functions handling that part of the request. - sharing = {} - for request in requests.values(): - member_of = request.member_of - for rc_name, amount in request.resources.items(): - rc_id = _RC_CACHE.id_from_string(rc_name) - if rc_id not in sharing: - sharing[rc_id] = _get_providers_with_shared_capacity( - context, rc_id, amount, member_of) - has_trees = _has_provider_trees(context) - - candidates = {} - for suffix, request in requests.items(): - alloc_reqs, summaries = cls._get_by_one_request( - context, request, sharing, has_trees) - LOG.debug("%s (suffix '%s') returned %d matches", - str(request), str(suffix), len(alloc_reqs)) - if not alloc_reqs: - # Shortcut: If any one request resulted in no candidates, the - # whole operation is shot. - return [], [] - # Mark each allocation request according to whether its - # corresponding RequestGroup required it to be restricted to a - # single provider. We'll need this later to evaluate group_policy. - for areq in alloc_reqs: - areq.use_same_provider = request.use_same_provider - candidates[suffix] = alloc_reqs, summaries - - # At this point, each (alloc_requests, summary_obj) in `candidates` is - # independent of the others. We need to fold them together such that - # each allocation request satisfies *all* the incoming `requests`. The - # `candidates` dict is guaranteed to contain entries for all suffixes, - # or we would have short-circuited above. - alloc_request_objs, summary_objs = _merge_candidates( - candidates, group_policy=group_policy) - - # Limit the number of allocation request objects. We do this after - # creating all of them so that we can do a random slice without - # needing to mess with the complex sql above or add additional - # columns to the DB. - if limit and limit <= len(alloc_request_objs): - if CONF.placement.randomize_allocation_candidates: - alloc_request_objs = random.sample(alloc_request_objs, limit) - else: - alloc_request_objs = alloc_request_objs[:limit] - elif CONF.placement.randomize_allocation_candidates: - random.shuffle(alloc_request_objs) - - # Limit summaries to only those mentioned in the allocation requests. - if limit and limit <= len(alloc_request_objs): - kept_summary_objs = [] - alloc_req_rp_uuids = set() - # Extract resource provider uuids from the resource requests. - for aro in alloc_request_objs: - for arr in aro.resource_requests: - alloc_req_rp_uuids.add(arr.resource_provider.uuid) - for summary in summary_objs: - rp_uuid = summary.resource_provider.uuid - # Skip a summary if we are limiting and haven't selected an - # allocation request that uses the resource provider. - if rp_uuid not in alloc_req_rp_uuids: - continue - kept_summary_objs.append(summary) - else: - kept_summary_objs = summary_objs - - return alloc_request_objs, kept_summary_objs - - -@db_api.placement_context_manager.writer -def reshape(ctx, inventories, allocations): - """The 'replace the world' strategy that is executed when we want to - completely replace a set of provider inventory, allocation and consumer - information in a single transaction. - - :note: The reason this has to be done in a single monolithic function is so - we have a single top-level function on which to decorate with the - @db_api.placement_context_manager.writer transaction context - manager. Each time a top-level function that is decorated with this - exits, the transaction is either COMMIT'd or ROLLBACK'd. We need to - avoid calling two functions that are already decorated with a - transaction context manager from a function that *isn't* decorated - with the transaction context manager if we want all changes involved - in the sub-functions to operate within a single DB transaction. - - :param ctx: `nova.api.openstack.placement.context.RequestContext` object - containing the DB transaction context. - :param inventories: dict, keyed by ResourceProvider, of `InventoryList` - objects representing the replaced inventory information - for the provider. - :param allocations: `AllocationList` object containing all allocations for - all consumers being modified by the reshape operation. - :raises: `exception.ConcurrentUpdateDetected` when any resource provider or - consumer generation increment fails due to concurrent changes to - the same objects. - """ - # The resource provider objects, keyed by provider UUID, that are involved - # in this transaction. We keep a cache of these because as we perform the - # various operations on the providers, their generations increment and we - # want to "inject" the changed resource provider objects into the - # AllocationList's objects before calling AllocationList.replace_all(). - # We start with the providers in the allocation objects, but only use one - # if we don't find it in the inventories. - affected_providers = {alloc.resource_provider.uuid: alloc.resource_provider - for alloc in allocations} - # We have to do the inventory changes in two steps because: - # - we can't delete inventories with allocations; and - # - we can't create allocations on nonexistent inventories. - # So in the first step we create a kind of "union" inventory for each - # provider. It contains all the inventories that the request wishes to - # exist in the end, PLUS any inventories that the request wished to remove - # (in their original form). - # Note that this can cause us to end up with an interim situation where we - # have modified an inventory to have less capacity than is currently - # allocated, but that's allowed by the code. If the final picture is - # overcommitted, we'll get an appropriate exception when we replace the - # allocations at the end. - for rp, new_inv_list in inventories.items(): - LOG.debug("reshaping: *interim* inventory replacement for provider %s", - rp.uuid) - # Update the cache. This may be replacing an entry that came from - # allocations, or adding a new entry from inventories. - affected_providers[rp.uuid] = rp - - # Optimization: If the new inventory is empty, the below would be - # replacing it with itself (and incrementing the generation) - # unnecessarily. - if not new_inv_list: - continue - - # A dict, keyed by resource class, of the Inventory objects. We start - # with the original inventory list. - inv_by_rc = {inv.resource_class: inv for inv in - InventoryList.get_all_by_resource_provider(ctx, rp)} - # Now add each inventory in the new inventory list. If an inventory for - # that resource class existed in the original inventory list, it is - # overwritten. - for inv in new_inv_list: - inv_by_rc[inv.resource_class] = inv - # Set the interim inventory structure. - rp.set_inventory(InventoryList(objects=list(inv_by_rc.values()))) - - # NOTE(jaypipes): The above inventory replacements will have - # incremented the resource provider generations, so we need to look in - # the AllocationList and swap the resource provider object with the one we - # saved above that has the updated provider generation in it. - for alloc in allocations: - rp_uuid = alloc.resource_provider.uuid - if rp_uuid in affected_providers: - alloc.resource_provider = affected_providers[rp_uuid] - - # Now we can replace all the allocations - LOG.debug("reshaping: attempting allocation replacement") - allocations.replace_all() - - # And finally, we can set the inventories to their actual desired state. - for rp, new_inv_list in inventories.items(): - LOG.debug("reshaping: *final* inventory replacement for provider %s", - rp.uuid) - rp.set_inventory(new_inv_list) diff --git a/nova/api/openstack/placement/objects/user.py b/nova/api/openstack/placement/objects/user.py deleted file mode 100644 index 8d5af8473d6e..000000000000 --- a/nova/api/openstack/placement/objects/user.py +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_db import exception as db_exc -from oslo_versionedobjects import base -from oslo_versionedobjects import fields -import sqlalchemy as sa - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import exception -from nova.db.sqlalchemy import api_models as models - -CONF = cfg.CONF -USER_TBL = models.User.__table__ - - -@db_api.placement_context_manager.writer -def ensure_incomplete_user(ctx): - """Ensures that a user record is created for the "incomplete consumer - user". Returns the internal ID of that record. - """ - incomplete_id = CONF.placement.incomplete_consumer_user_id - sel = sa.select([USER_TBL.c.id]).where( - USER_TBL.c.external_id == incomplete_id) - res = ctx.session.execute(sel).fetchone() - if res: - return res[0] - ins = USER_TBL.insert().values(external_id=incomplete_id) - res = ctx.session.execute(ins) - return res.inserted_primary_key[0] - - -@db_api.placement_context_manager.reader -def _get_user_by_external_id(ctx, external_id): - users = sa.alias(USER_TBL, name="u") - cols = [ - users.c.id, - users.c.external_id, - users.c.updated_at, - users.c.created_at - ] - sel = sa.select(cols) - sel = sel.where(users.c.external_id == external_id) - res = ctx.session.execute(sel).fetchone() - if not res: - raise exception.UserNotFound(external_id=external_id) - - return dict(res) - - -@base.VersionedObjectRegistry.register_if(False) -class User(base.VersionedObject): - - fields = { - 'id': fields.IntegerField(read_only=True), - 'external_id': fields.StringField(nullable=False), - } - - @staticmethod - def _from_db_object(ctx, target, source): - for field in target.fields: - setattr(target, field, source[field]) - - target._context = ctx - target.obj_reset_changes() - return target - - @classmethod - def get_by_external_id(cls, ctx, external_id): - res = _get_user_by_external_id(ctx, external_id) - return cls._from_db_object(ctx, cls(ctx), res) - - def create(self): - @db_api.placement_context_manager.writer - def _create_in_db(ctx): - db_obj = models.User(external_id=self.external_id) - try: - db_obj.save(ctx.session) - except db_exc.DBDuplicateEntry: - raise exception.UserExists(external_id=self.external_id) - self._from_db_object(ctx, self, db_obj) - _create_in_db(self._context) diff --git a/nova/api/openstack/placement/policies/__init__.py b/nova/api/openstack/placement/policies/__init__.py deleted file mode 100644 index be0496d23b5d..000000000000 --- a/nova/api/openstack/placement/policies/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import itertools - -from nova.api.openstack.placement.policies import aggregate -from nova.api.openstack.placement.policies import allocation -from nova.api.openstack.placement.policies import allocation_candidate -from nova.api.openstack.placement.policies import base -from nova.api.openstack.placement.policies import inventory -from nova.api.openstack.placement.policies import reshaper -from nova.api.openstack.placement.policies import resource_class -from nova.api.openstack.placement.policies import resource_provider -from nova.api.openstack.placement.policies import trait -from nova.api.openstack.placement.policies import usage - - -def list_rules(): - return itertools.chain( - base.list_rules(), - resource_provider.list_rules(), - resource_class.list_rules(), - inventory.list_rules(), - aggregate.list_rules(), - usage.list_rules(), - trait.list_rules(), - allocation.list_rules(), - allocation_candidate.list_rules(), - reshaper.list_rules(), - ) diff --git a/nova/api/openstack/placement/policies/aggregate.py b/nova/api/openstack/placement/policies/aggregate.py deleted file mode 100644 index 8e2bd8c3ab76..000000000000 --- a/nova/api/openstack/placement/policies/aggregate.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PREFIX = 'placement:resource_providers:aggregates:%s' -LIST = PREFIX % 'list' -UPDATE = PREFIX % 'update' -BASE_PATH = '/resource_providers/{uuid}/aggregates' - -rules = [ - policy.DocumentedRuleDefault( - LIST, - base.RULE_ADMIN_API, - "List resource provider aggregates.", - [ - { - 'method': 'GET', - 'path': BASE_PATH - } - ], - scope_types=['system'] - ), - policy.DocumentedRuleDefault( - UPDATE, - base.RULE_ADMIN_API, - "Update resource provider aggregates.", - [ - { - 'method': 'PUT', - 'path': BASE_PATH - } - ], - scope_types=['system'] - ), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/allocation.py b/nova/api/openstack/placement/policies/allocation.py deleted file mode 100644 index a5f1c2e0017d..000000000000 --- a/nova/api/openstack/placement/policies/allocation.py +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -RP_ALLOC_LIST = 'placement:resource_providers:allocations:list' - -ALLOC_PREFIX = 'placement:allocations:%s' -ALLOC_LIST = ALLOC_PREFIX % 'list' -ALLOC_MANAGE = ALLOC_PREFIX % 'manage' -ALLOC_UPDATE = ALLOC_PREFIX % 'update' -ALLOC_DELETE = ALLOC_PREFIX % 'delete' - -rules = [ - policy.DocumentedRuleDefault( - ALLOC_MANAGE, - base.RULE_ADMIN_API, - "Manage allocations.", - [ - { - 'method': 'POST', - 'path': '/allocations' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - ALLOC_LIST, - base.RULE_ADMIN_API, - "List allocations.", - [ - { - 'method': 'GET', - 'path': '/allocations/{consumer_uuid}' - } - ], - scope_types=['system'] - ), - policy.DocumentedRuleDefault( - ALLOC_UPDATE, - base.RULE_ADMIN_API, - "Update allocations.", - [ - { - 'method': 'PUT', - 'path': '/allocations/{consumer_uuid}' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - ALLOC_DELETE, - base.RULE_ADMIN_API, - "Delete allocations.", - [ - { - 'method': 'DELETE', - 'path': '/allocations/{consumer_uuid}' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - RP_ALLOC_LIST, - base.RULE_ADMIN_API, - "List resource provider allocations.", - [ - { - 'method': 'GET', - 'path': '/resource_providers/{uuid}/allocations' - } - ], - scope_types=['system'], - ), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/allocation_candidate.py b/nova/api/openstack/placement/policies/allocation_candidate.py deleted file mode 100644 index e2ae655370d6..000000000000 --- a/nova/api/openstack/placement/policies/allocation_candidate.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -LIST = 'placement:allocation_candidates:list' - -rules = [ - policy.DocumentedRuleDefault( - LIST, - base.RULE_ADMIN_API, - "List allocation candidates.", - [ - { - 'method': 'GET', - 'path': '/allocation_candidates' - } - ], - scope_types=['system'], - ) -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/base.py b/nova/api/openstack/placement/policies/base.py deleted file mode 100644 index 1e728a37fad1..000000000000 --- a/nova/api/openstack/placement/policies/base.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_policy import policy - -RULE_ADMIN_API = 'rule:admin_api' - -rules = [ - # "placement" is the default rule (action) used for all routes that do - # not yet have granular policy rules. It is used in - # PlacementHandler.__call__ and can be dropped once all routes have - # granular policy handling. - policy.RuleDefault( - "placement", - "role:admin", - description="This rule is used for all routes that do not yet " - "have granular policy rules. It will be replaced " - "with rule:admin_api.", - deprecated_for_removal=True, - deprecated_reason="This was a catch-all rule hard-coded into " - "the placement service and has been superseded by " - "granular policy rules per operation.", - deprecated_since="18.0.0"), - policy.RuleDefault( - "admin_api", - "role:admin", - description="Default rule for most placement APIs.", - scope_types=['system']), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/inventory.py b/nova/api/openstack/placement/policies/inventory.py deleted file mode 100644 index 1f3d38f413fa..000000000000 --- a/nova/api/openstack/placement/policies/inventory.py +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PREFIX = 'placement:resource_providers:inventories:%s' -LIST = PREFIX % 'list' -CREATE = PREFIX % 'create' -SHOW = PREFIX % 'show' -UPDATE = PREFIX % 'update' -DELETE = PREFIX % 'delete' -BASE_PATH = '/resource_providers/{uuid}/inventories' - -rules = [ - policy.DocumentedRuleDefault( - LIST, - base.RULE_ADMIN_API, - "List resource provider inventories.", - [ - { - 'method': 'GET', - 'path': BASE_PATH - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - CREATE, - base.RULE_ADMIN_API, - "Create one resource provider inventory.", - [ - { - 'method': 'POST', - 'path': BASE_PATH - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - SHOW, - base.RULE_ADMIN_API, - "Show resource provider inventory.", - [ - { - 'method': 'GET', - 'path': BASE_PATH + '/{resource_class}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - UPDATE, - base.RULE_ADMIN_API, - "Update resource provider inventory.", - [ - { - 'method': 'PUT', - 'path': BASE_PATH - }, - { - 'method': 'PUT', - 'path': BASE_PATH + '/{resource_class}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - DELETE, - base.RULE_ADMIN_API, - "Delete resource provider inventory.", - [ - { - 'method': 'DELETE', - 'path': BASE_PATH - }, - { - 'method': 'DELETE', - 'path': BASE_PATH + '/{resource_class}' - } - ], - scope_types=['system']), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/reshaper.py b/nova/api/openstack/placement/policies/reshaper.py deleted file mode 100644 index a6615ac48752..000000000000 --- a/nova/api/openstack/placement/policies/reshaper.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PREFIX = 'placement:reshaper:%s' -RESHAPE = PREFIX % 'reshape' - -rules = [ - policy.DocumentedRuleDefault( - RESHAPE, - base.RULE_ADMIN_API, - "Reshape Inventory and Allocations.", - [ - { - 'method': 'POST', - 'path': '/reshaper' - } - ], - scope_types=['system']), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/resource_class.py b/nova/api/openstack/placement/policies/resource_class.py deleted file mode 100644 index 75acab9d3b7b..000000000000 --- a/nova/api/openstack/placement/policies/resource_class.py +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PREFIX = 'placement:resource_classes:%s' -LIST = PREFIX % 'list' -CREATE = PREFIX % 'create' -SHOW = PREFIX % 'show' -UPDATE = PREFIX % 'update' -DELETE = PREFIX % 'delete' - -rules = [ - policy.DocumentedRuleDefault( - LIST, - base.RULE_ADMIN_API, - "List resource classes.", - [ - { - 'method': 'GET', - 'path': '/resource_classes' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - CREATE, - base.RULE_ADMIN_API, - "Create resource class.", - [ - { - 'method': 'POST', - 'path': '/resource_classes' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - SHOW, - base.RULE_ADMIN_API, - "Show resource class.", - [ - { - 'method': 'GET', - 'path': '/resource_classes/{name}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - UPDATE, - base.RULE_ADMIN_API, - "Update resource class.", - [ - { - 'method': 'PUT', - 'path': '/resource_classes/{name}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - DELETE, - base.RULE_ADMIN_API, - "Delete resource class.", - [ - { - 'method': 'DELETE', - 'path': '/resource_classes/{name}' - } - ], - scope_types=['system']), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/resource_provider.py b/nova/api/openstack/placement/policies/resource_provider.py deleted file mode 100644 index 7c4826bd705e..000000000000 --- a/nova/api/openstack/placement/policies/resource_provider.py +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PREFIX = 'placement:resource_providers:%s' -LIST = PREFIX % 'list' -CREATE = PREFIX % 'create' -SHOW = PREFIX % 'show' -UPDATE = PREFIX % 'update' -DELETE = PREFIX % 'delete' - -rules = [ - policy.DocumentedRuleDefault( - LIST, - base.RULE_ADMIN_API, - "List resource providers.", - [ - { - 'method': 'GET', - 'path': '/resource_providers' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - CREATE, - base.RULE_ADMIN_API, - "Create resource provider.", - [ - { - 'method': 'POST', - 'path': '/resource_providers' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - SHOW, - base.RULE_ADMIN_API, - "Show resource provider.", - [ - { - 'method': 'GET', - 'path': '/resource_providers/{uuid}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - UPDATE, - base.RULE_ADMIN_API, - "Update resource provider.", - [ - { - 'method': 'PUT', - 'path': '/resource_providers/{uuid}' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - DELETE, - base.RULE_ADMIN_API, - "Delete resource provider.", - [ - { - 'method': 'DELETE', - 'path': '/resource_providers/{uuid}' - } - ], - scope_types=['system']), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/trait.py b/nova/api/openstack/placement/policies/trait.py deleted file mode 100644 index 6b35a703de5c..000000000000 --- a/nova/api/openstack/placement/policies/trait.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -RP_TRAIT_PREFIX = 'placement:resource_providers:traits:%s' -RP_TRAIT_LIST = RP_TRAIT_PREFIX % 'list' -RP_TRAIT_UPDATE = RP_TRAIT_PREFIX % 'update' -RP_TRAIT_DELETE = RP_TRAIT_PREFIX % 'delete' - -TRAITS_PREFIX = 'placement:traits:%s' -TRAITS_LIST = TRAITS_PREFIX % 'list' -TRAITS_SHOW = TRAITS_PREFIX % 'show' -TRAITS_UPDATE = TRAITS_PREFIX % 'update' -TRAITS_DELETE = TRAITS_PREFIX % 'delete' - - -rules = [ - policy.DocumentedRuleDefault( - TRAITS_LIST, - base.RULE_ADMIN_API, - "List traits.", - [ - { - 'method': 'GET', - 'path': '/traits' - } - ], - scope_types=['system'] - ), - policy.DocumentedRuleDefault( - TRAITS_SHOW, - base.RULE_ADMIN_API, - "Show trait.", - [ - { - 'method': 'GET', - 'path': '/traits/{name}' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - TRAITS_UPDATE, - base.RULE_ADMIN_API, - "Update trait.", - [ - { - 'method': 'PUT', - 'path': '/traits/{name}' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - TRAITS_DELETE, - base.RULE_ADMIN_API, - "Delete trait.", - [ - { - 'method': 'DELETE', - 'path': '/traits/{name}' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - RP_TRAIT_LIST, - base.RULE_ADMIN_API, - "List resource provider traits.", - [ - { - 'method': 'GET', - 'path': '/resource_providers/{uuid}/traits' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - RP_TRAIT_UPDATE, - base.RULE_ADMIN_API, - "Update resource provider traits.", - [ - { - 'method': 'PUT', - 'path': '/resource_providers/{uuid}/traits' - } - ], - scope_types=['system'], - ), - policy.DocumentedRuleDefault( - RP_TRAIT_DELETE, - base.RULE_ADMIN_API, - "Delete resource provider traits.", - [ - { - 'method': 'DELETE', - 'path': '/resource_providers/{uuid}/traits' - } - ], - scope_types=['system'], - ), -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policies/usage.py b/nova/api/openstack/placement/policies/usage.py deleted file mode 100644 index 6543fa4359d5..000000000000 --- a/nova/api/openstack/placement/policies/usage.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_policy import policy - -from nova.api.openstack.placement.policies import base - - -PROVIDER_USAGES = 'placement:resource_providers:usages' -TOTAL_USAGES = 'placement:usages' - - -rules = [ - policy.DocumentedRuleDefault( - PROVIDER_USAGES, - base.RULE_ADMIN_API, - "List resource provider usages.", - [ - { - 'method': 'GET', - 'path': '/resource_providers/{uuid}/usages' - } - ], - scope_types=['system']), - policy.DocumentedRuleDefault( - # TODO(mriedem): At some point we might set scope_types=['project'] - # so that non-admin project-scoped token users can query usages for - # their project. The context.can() target will need to change as well - # in the actual policy enforcement check in the handler code. - TOTAL_USAGES, - base.RULE_ADMIN_API, - "List total resource usages for a given project.", - [ - { - 'method': 'GET', - 'path': '/usages' - } - ], - scope_types=['system']) -] - - -def list_rules(): - return rules diff --git a/nova/api/openstack/placement/policy.py b/nova/api/openstack/placement/policy.py deleted file mode 100644 index cad6fdf8388b..000000000000 --- a/nova/api/openstack/placement/policy.py +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Policy Enforcement for placement API.""" - -from oslo_config import cfg -from oslo_log import log as logging -from oslo_policy import policy -from oslo_utils import excutils - -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import policies - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) -_ENFORCER_PLACEMENT = None - - -def reset(): - """Used to reset the global _ENFORCER_PLACEMENT between test runs.""" - global _ENFORCER_PLACEMENT - if _ENFORCER_PLACEMENT: - _ENFORCER_PLACEMENT.clear() - _ENFORCER_PLACEMENT = None - - -def init(): - """Init an Enforcer class. Sets the _ENFORCER_PLACEMENT global.""" - global _ENFORCER_PLACEMENT - if not _ENFORCER_PLACEMENT: - # NOTE(mriedem): We have to explicitly pass in the - # [placement]/policy_file path because otherwise oslo_policy defaults - # to read the policy file from config option [oslo_policy]/policy_file - # which is used by nova. In other words, to have separate policy files - # for placement and nova, we have to use separate policy_file options. - _ENFORCER_PLACEMENT = policy.Enforcer( - CONF, policy_file=CONF.placement.policy_file) - _ENFORCER_PLACEMENT.register_defaults(policies.list_rules()) - _ENFORCER_PLACEMENT.load_rules() - - -def get_enforcer(): - # This method is used by oslopolicy CLI scripts in order to generate policy - # files from overrides on disk and defaults in code. We can just pass an - # empty list and let oslo do the config lifting for us. - # TODO(mriedem): Change the project kwarg value to "placement" once - # this code is extracted from nova. - cfg.CONF([], project='nova') - init() - return _ENFORCER_PLACEMENT - - -def authorize(context, action, target, do_raise=True): - """Verifies that the action is valid on the target in this context. - - :param context: instance of - nova.api.openstack.placement.context.RequestContext - :param action: string representing the action to be checked - this should be colon separated for clarity, i.e. - ``placement:resource_providers:list`` - :param target: dictionary representing the object of the action; - for object creation this should be a dictionary representing the - owner of the object e.g. ``{'project_id': context.project_id}``. - :param do_raise: if True (the default), raises PolicyNotAuthorized; - if False, returns False - :raises nova.api.openstack.placement.exception.PolicyNotAuthorized: if - verification fails and do_raise is True. - :returns: non-False value (not necessarily "True") if authorized, and the - exact value False if not authorized and do_raise is False. - """ - init() - credentials = context.to_policy_values() - try: - # NOTE(mriedem): The "action" kwarg is for the PolicyNotAuthorized exc. - return _ENFORCER_PLACEMENT.authorize( - action, target, credentials, do_raise=do_raise, - exc=exception.PolicyNotAuthorized, action=action) - except policy.PolicyNotRegistered: - with excutils.save_and_reraise_exception(): - LOG.exception('Policy not registered') - except Exception: - with excutils.save_and_reraise_exception(): - LOG.debug('Policy check for %(action)s failed with credentials ' - '%(credentials)s', - {'action': action, 'credentials': credentials}) diff --git a/nova/api/openstack/placement/requestlog.py b/nova/api/openstack/placement/requestlog.py deleted file mode 100644 index da7be6a37f4a..000000000000 --- a/nova/api/openstack/placement/requestlog.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Simple middleware for request logging.""" - -from oslo_log import log as logging - -from nova.api.openstack.placement import microversion - -LOG = logging.getLogger(__name__) - - -class RequestLog(object): - """WSGI Middleware to write a simple request log to. - - Borrowed from Paste Translogger - """ - - format = ('%(REMOTE_ADDR)s "%(REQUEST_METHOD)s %(REQUEST_URI)s" ' - 'status: %(status)s len: %(bytes)s ' - 'microversion: %(microversion)s') - - def __init__(self, application): - self.application = application - - def __call__(self, environ, start_response): - LOG.debug('Starting request: %s "%s %s"', - environ['REMOTE_ADDR'], environ['REQUEST_METHOD'], - self._get_uri(environ)) - # Set the accept header if it is not otherwise set or is '*/*'. This - # ensures that error responses will be in JSON. - accept = environ.get('HTTP_ACCEPT') - if not accept or accept == '*/*': - environ['HTTP_ACCEPT'] = 'application/json' - if LOG.isEnabledFor(logging.INFO): - return self._log_app(environ, start_response) - else: - return self.application(environ, start_response) - - @staticmethod - def _get_uri(environ): - req_uri = (environ.get('SCRIPT_NAME', '') - + environ.get('PATH_INFO', '')) - if environ.get('QUERY_STRING'): - req_uri += '?' + environ['QUERY_STRING'] - return req_uri - - def _log_app(self, environ, start_response): - req_uri = self._get_uri(environ) - - def replacement_start_response(status, headers, exc_info=None): - """We need to gaze at the content-length, if set, to - write log info. - """ - size = None - for name, value in headers: - if name.lower() == 'content-length': - size = value - self.write_log(environ, req_uri, status, size) - return start_response(status, headers, exc_info) - - return self.application(environ, replacement_start_response) - - def write_log(self, environ, req_uri, status, size): - """Write the log info out in a formatted form to ``LOG.info``. - """ - if size is None: - size = '-' - log_format = { - 'REMOTE_ADDR': environ.get('REMOTE_ADDR', '-'), - 'REQUEST_METHOD': environ['REQUEST_METHOD'], - 'REQUEST_URI': req_uri, - 'status': status.split(None, 1)[0], - 'bytes': size, - 'microversion': environ.get( - microversion.MICROVERSION_ENVIRON, '-'), - } - LOG.info(self.format, log_format) diff --git a/nova/api/openstack/placement/resource_class_cache.py b/nova/api/openstack/placement/resource_class_cache.py deleted file mode 100644 index a72b4177ea6a..000000000000 --- a/nova/api/openstack/placement/resource_class_cache.py +++ /dev/null @@ -1,154 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_concurrency import lockutils -import sqlalchemy as sa - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import exception -from nova.db.sqlalchemy import api_models as models -from nova import rc_fields as fields - -_RC_TBL = models.ResourceClass.__table__ -_LOCKNAME = 'rc_cache' - - -@db_api.placement_context_manager.reader -def _refresh_from_db(ctx, cache): - """Grabs all custom resource classes from the DB table and populates the - supplied cache object's internal integer and string identifier dicts. - - :param cache: ResourceClassCache object to refresh. - """ - with db_api.placement_context_manager.reader.connection.using(ctx) as conn: - sel = sa.select([_RC_TBL.c.id, _RC_TBL.c.name, _RC_TBL.c.updated_at, - _RC_TBL.c.created_at]) - res = conn.execute(sel).fetchall() - cache.id_cache = {r[1]: r[0] for r in res} - cache.str_cache = {r[0]: r[1] for r in res} - cache.all_cache = {r[1]: r for r in res} - - -class ResourceClassCache(object): - """A cache of integer and string lookup values for resource classes.""" - - # List of dict of all standard resource classes, where every list item - # have a form {'id': , 'name': } - STANDARDS = [{'id': fields.ResourceClass.STANDARD.index(s), 'name': s, - 'updated_at': None, 'created_at': None} - for s in fields.ResourceClass.STANDARD] - - def __init__(self, ctx): - """Initialize the cache of resource class identifiers. - - :param ctx: `nova.context.RequestContext` from which we can grab a - `SQLAlchemy.Connection` object to use for any DB lookups. - """ - self.ctx = ctx - self.id_cache = {} - self.str_cache = {} - self.all_cache = {} - - def clear(self): - with lockutils.lock(_LOCKNAME): - self.id_cache = {} - self.str_cache = {} - self.all_cache = {} - - def id_from_string(self, rc_str): - """Given a string representation of a resource class -- e.g. "DISK_GB" - or "IRON_SILVER" -- return the integer code for the resource class. For - standard resource classes, this integer code will match the list of - resource classes on the fields.ResourceClass field type. Other custom - resource classes will cause a DB lookup into the resource_classes - table, however the results of these DB lookups are cached since the - lookups are so frequent. - - :param rc_str: The string representation of the resource class to look - up a numeric identifier for. - :returns integer identifier for the resource class, or None, if no such - resource class was found in the list of standard resource - classes or the resource_classes database table. - :raises `exception.ResourceClassNotFound` if rc_str cannot be found in - either the standard classes or the DB. - """ - # First check the standard resource classes - if rc_str in fields.ResourceClass.STANDARD: - return fields.ResourceClass.STANDARD.index(rc_str) - - with lockutils.lock(_LOCKNAME): - if rc_str in self.id_cache: - return self.id_cache[rc_str] - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_str in self.id_cache: - return self.id_cache[rc_str] - raise exception.ResourceClassNotFound(resource_class=rc_str) - - def all_from_string(self, rc_str): - """Given a string representation of a resource class -- e.g. "DISK_GB" - or "CUSTOM_IRON_SILVER" -- return all the resource class info. - - :param rc_str: The string representation of the resource class for - which to look up a resource_class. - :returns: dict representing the resource class fields, if the - resource class was found in the list of standard - resource classes or the resource_classes database table. - :raises: `exception.ResourceClassNotFound` if rc_str cannot be found in - either the standard classes or the DB. - """ - # First check the standard resource classes - if rc_str in fields.ResourceClass.STANDARD: - return {'id': fields.ResourceClass.STANDARD.index(rc_str), - 'name': rc_str, - 'updated_at': None, - 'created_at': None} - - with lockutils.lock(_LOCKNAME): - if rc_str in self.all_cache: - return self.all_cache[rc_str] - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_str in self.all_cache: - return self.all_cache[rc_str] - raise exception.ResourceClassNotFound(resource_class=rc_str) - - def string_from_id(self, rc_id): - """The reverse of the id_from_string() method. Given a supplied numeric - identifier for a resource class, we look up the corresponding string - representation, either in the list of standard resource classes or via - a DB lookup. The results of these DB lookups are cached since the - lookups are so frequent. - - :param rc_id: The numeric representation of the resource class to look - up a string identifier for. - :returns: string identifier for the resource class, or None, if no such - resource class was found in the list of standard resource - classes or the resource_classes database table. - :raises `exception.ResourceClassNotFound` if rc_id cannot be found in - either the standard classes or the DB. - """ - # First check the fields.ResourceClass.STANDARD values - try: - return fields.ResourceClass.STANDARD[rc_id] - except IndexError: - pass - - with lockutils.lock(_LOCKNAME): - if rc_id in self.str_cache: - return self.str_cache[rc_id] - - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_id in self.str_cache: - return self.str_cache[rc_id] - raise exception.ResourceClassNotFound(resource_class=rc_id) diff --git a/nova/api/openstack/placement/rest_api_version_history.rst b/nova/api/openstack/placement/rest_api_version_history.rst deleted file mode 100644 index 10bd180195d6..000000000000 --- a/nova/api/openstack/placement/rest_api_version_history.rst +++ /dev/null @@ -1,518 +0,0 @@ -REST API Version History -~~~~~~~~~~~~~~~~~~~~~~~~ - -This documents the changes made to the REST API with every microversion change. -The description for each version should be a verbose one which has enough -information to be suitable for use in user documentation. - -.. _1.0 (Maximum in Newton): - -1.0 Initial Version (Maximum in Newton) ---------------------------------------- - -.. versionadded:: Newton - -This is the initial version of the placement REST API that was released in -Nova 14.0.0 (Newton). This contains the following routes: - -* ``/resource_providers`` -* ``/resource_providers/allocations`` -* ``/resource_providers/inventories`` -* ``/resource_providers/usages`` -* ``/allocations`` - -1.1 Resource provider aggregates --------------------------------- - -.. versionadded:: Ocata - -The 1.1 version adds support for associating aggregates with resource -providers. - -The following new operations are added: - -``GET /resource_providers/{uuid}/aggregates`` - Return all aggregates associated with a resource provider - -``PUT /resource_providers/{uuid}/aggregates`` - Update the aggregates associated with a resource provider - -1.2 Add custom resource classes -------------------------------- - -.. versionadded:: Ocata - -Placement API version 1.2 adds basic operations allowing an admin to create, -list and delete custom resource classes. - -The following new routes are added: - -``GET /resource_classes`` - Return all resource classes - -``POST /resource_classes`` - Create a new custom resource class - -``PUT /resource_classes/{name}`` - Update the name of a custom resource class - -``DELETE /resource_classes/{name}`` - Delete a custom resource class - -``GET /resource_classes/{name}`` - Get a single resource class - -Custom resource classes must begin with the prefix ``CUSTOM_`` and contain only -the letters A through Z, the numbers 0 through 9 and the underscore ``_`` -character. - -1.3 member_of query parameter ------------------------------ - -.. versionadded:: Ocata - -Version 1.3 adds support for listing resource providers that are members of any -of the list of aggregates provided using a ``member_of`` query parameter:: - - ?member_of=in:{agg1_uuid},{agg2_uuid},{agg3_uuid} - -1.4 Filter resource providers by requested resource capacity (Maximum in Ocata) -------------------------------------------------------------------------------- - -.. versionadded:: Ocata - -The 1.4 version adds support for querying resource providers that have the -ability to serve a requested set of resources. A new "resources" query string -parameter is now accepted to the ``GET /resource_providers`` API call. This -parameter indicates the requested amounts of various resources that a provider -must have the capacity to serve. The "resources" query string parameter takes -the form:: - - ?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT - -For instance, if the user wishes to see resource providers that can service a -request for 2 vCPUs, 1024 MB of RAM and 50 GB of disk space, the user can issue -a request to:: - - GET /resource_providers?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50 - -If the resource class does not exist, then it will return a HTTP 400. - -.. note:: The resources filtering is also based on the `min_unit`, `max_unit` - and `step_size` of the inventory record. For example, if the `max_unit` is - 512 for the DISK_GB inventory for a particular resource provider and a - GET request is made for `DISK_GB:1024`, that resource provider will not be - returned. The `min_unit` is the minimum amount of resource that can be - requested for a given inventory and resource provider. The `step_size` is - the increment of resource that can be requested for a given resource on a - given provider. - -1.5 DELETE all inventory for a resource provider ------------------------------------------------- - -.. versionadded:: Pike - -Placement API version 1.5 adds DELETE method for deleting all inventory for a -resource provider. The following new method is supported: - -``DELETE /resource_providers/{uuid}/inventories`` - - Delete all inventories for a given resource provider - -1.6 Traits API --------------- - -.. versionadded:: Pike - -The 1.6 version adds basic operations allowing an admin to create, list, and -delete custom traits, also adds basic operations allowing an admin to attach -traits to a resource provider. - -The following new routes are added: - -``GET /traits`` - Return all resource classes. - -``PUT /traits/{name}`` - Insert a single custom trait. - -``GET /traits/{name}`` - Check if a trait name exists. - -``DELETE /traits/{name}`` - Delete the specified trait. - -``GET /resource_providers/{uuid}/traits`` - Return all traits associated with a specific resource provider. - -``PUT /resource_providers/{uuid}/traits`` - Update all traits for a specific resource provider. - -``DELETE /resource_providers/{uuid}/traits`` - Remove any existing trait associations for a specific resource provider - -Custom traits must begin with the prefix ``CUSTOM_`` and contain only the -letters A through Z, the numbers 0 through 9 and the underscore ``_`` -character. - -1.7 Idempotent PUT /resource_classes/{name} -------------------------------------------- - -.. versionadded:: Pike - -The 1.7 version changes handling of ``PUT /resource_classes/{name}`` to be a -create or verification of the resource class with ``{name}``. If the resource -class is a custom resource class and does not already exist it will be created -and a ``201`` response code returned. If the class already exists the response -code will be ``204``. This makes it possible to check or create a resource -class in one request. - -1.8 Require placement 'project_id', 'user_id' in PUT /allocations ------------------------------------------------------------------ - -.. versionadded:: Pike - -The 1.8 version adds ``project_id`` and ``user_id`` required request parameters -to ``PUT /allocations``. - -1.9 Add GET /usages --------------------- - -.. versionadded:: Pike - -The 1.9 version adds usages that can be queried by a project or project/user. - -The following new routes are added: - -``GET /usages?project_id=`` - Return all usages for a given project. - -``GET /usages?project_id=&user_id=`` - Return all usages for a given project and user. - -1.10 Allocation candidates (Maximum in Pike) --------------------------------------------- - -.. versionadded:: Pike - -The 1.10 version brings a new REST resource endpoint for getting a list of -allocation candidates. Allocation candidates are collections of possible -allocations against resource providers that can satisfy a particular request -for resources. - -1.11 Add 'allocations' link to the ``GET /resource_providers`` response ------------------------------------------------------------------------ - -.. versionadded:: Queens - -The ``/resource_providers/{rp_uuid}/allocations`` endpoint has been available -since version 1.0, but was not listed in the ``links`` section of the -``GET /resource_providers`` response. The link is included as of version 1.11. - -1.12 PUT dict format to /allocations/{consumer_uuid} ----------------------------------------------------- - -.. versionadded:: Queens - -In version 1.12 the request body of a ``PUT /allocations/{consumer_uuid}`` -is expected to have an ``object`` for the ``allocations`` property, not as -``array`` as with earlier microversions. This puts the request body more in -alignment with the structure of the ``GET /allocations/{consumer_uuid}`` -response body. Because the ``PUT`` request requires ``user_id`` and -``project_id`` in the request body, these fields are added to the ``GET`` -response. In addition, the response body for ``GET /allocation_candidates`` -is updated so the allocations in the ``alocation_requests`` object work -with the new ``PUT`` format. - -1.13 POST multiple allocations to /allocations ----------------------------------------------- - -.. versionadded:: Queens - -Version 1.13 gives the ability to set or clear allocations for more than -one consumer UUID with a request to ``POST /allocations``. - -1.14 Add nested resource providers ----------------------------------- - -.. versionadded:: Queens - -The 1.14 version introduces the concept of nested resource providers. The -resource provider resource now contains two new attributes: - -* ``parent_provider_uuid`` indicates the provider's direct parent, or null if - there is no parent. This attribute can be set in the call to ``POST - /resource_providers`` and ``PUT /resource_providers/{uuid}`` if the attribute - has not already been set to a non-NULL value (i.e. we do not support - "reparenting" a provider) -* ``root_provider_uuid`` indicates the UUID of the root resource provider in - the provider's tree. This is a read-only attribute - -A new ``in_tree=`` parameter is now available in the ``GET -/resource-providers`` API call. Supplying a UUID value for the ``in_tree`` -parameter will cause all resource providers within the "provider tree" of the -provider matching ```` to be returned. - -1.15 Add 'last-modified' and 'cache-control' headers ----------------------------------------------------- - -.. versionadded:: Queens - -Throughout the API, 'last-modified' headers have been added to GET responses -and those PUT and POST responses that have bodies. The value is either the -actual last modified time of the most recently modified associated database -entity or the current time if there is no direct mapping to the database. In -addition, 'cache-control: no-cache' headers are added where the 'last-modified' -header has been added to prevent inadvertent caching of resources. - -1.16 Limit allocation candidates --------------------------------- - -.. versionadded:: Queens - -Add support for a ``limit`` query parameter when making a -``GET /allocation_candidates`` request. The parameter accepts an integer -value, ``N``, which limits the maximum number of candidates returned. - -1.17 Add 'required' parameter to the allocation candidates (Maximum in Queens) ------------------------------------------------------------------------------- - -.. versionadded:: Queens - -Add the ``required`` parameter to the ``GET /allocation_candidates`` API. It -accepts a list of traits separated by ``,``. The provider summary in the -response will include the attached traits also. - -1.18 Support ?required= queryparam on GET /resource_providers ---------------------------------------------------------------------- - -.. versionadded:: Rocky - -Add support for the ``required`` query parameter to the ``GET -/resource_providers`` API. It accepts a comma-separated list of string trait -names. When specified, the API results will be filtered to include only -resource providers marked with all the specified traits. This is in addition to -(logical AND) any filtering based on other query parameters. - -Trait names which are empty, do not exist, or are otherwise invalid will result -in a 400 error. - -1.19 Include generation and conflict detection in provider aggregates APIs --------------------------------------------------------------------------- - -.. versionadded:: Rocky - -Enhance the payloads for the ``GET /resource_providers/{uuid}/aggregates`` -response and the ``PUT /resource_providers/{uuid}/aggregates`` request and -response to be identical, and to include the ``resource_provider_generation``. -As with other generation-aware APIs, if the ``resource_provider_generation`` -specified in the ``PUT`` request does not match the generation known by the -server, a 409 Conflict error is returned. - -1.20 Return 200 with provider payload from POST /resource_providers -------------------------------------------------------------------- - -.. versionadded:: Rocky - -The ``POST /resource_providers`` API, on success, returns 200 with a payload -representing the newly-created resource provider, in the same format as the -corresponding ``GET /resource_providers/{uuid}`` call. This is to allow the -caller to glean automatically-set fields, such as UUID and generation, without -a subsequent GET. - -1.21 Support ?member_of= queryparam on GET /allocation_candidates ------------------------------------------------------------------------------ - -.. versionadded:: Rocky - -Add support for the ``member_of`` query parameter to the ``GET -/allocation_candidates`` API. It accepts a comma-separated list of UUIDs for -aggregates. Note that if more than one aggregate UUID is passed, the -comma-separated list must be prefixed with the "in:" operator. If this -parameter is provided, the only resource providers returned will be those in -one of the specified aggregates that meet the other parts of the request. - -1.22 Support forbidden traits on resource providers and allocations candidates ------------------------------------------------------------------------------- - -.. versionadded:: Rocky - -Add support for expressing traits which are forbidden when filtering -``GET /resource_providers`` or ``GET /allocation_candidates``. A forbidden -trait is a properly formatted trait in the existing ``required`` parameter, -prefixed by a ``!``. For example ``required=!STORAGE_DISK_SSD`` asks that the -results not include any resource providers that provide solid state disk. - -1.23 Include code attribute in JSON error responses ---------------------------------------------------- - -.. versionadded:: Rocky - -JSON formatted error responses gain a new attribute, ``code``, with a value -that identifies the type of this error. This can be used to distinguish errors -that are different but use the same HTTP status code. Any error response which -does not specifically define a code will have the code -``placement.undefined_code``. - -1.24 Support multiple ?member_of queryparams --------------------------------------------- - -.. versionadded:: Rocky - -Add support for specifying multiple ``member_of`` query parameters to the ``GET -/resource_providers`` API. When multiple ``member_of`` query parameters are -found, they are AND'd together in the final query. For example, issuing a -request for ``GET /resource_providers?member_of=agg1&member_of=agg2`` means get -the resource providers that are associated with BOTH agg1 and agg2. Issuing a -request for ``GET /resource_providers?member_of=in:agg1,agg2&member_of=agg3`` -means get the resource providers that are associated with agg3 and are also -associated with *any of* (agg1, agg2). - -1.25 Granular resource requests to ``GET /allocation_candidates`` ------------------------------------------------------------------ - -.. versionadded:: Rocky - -``GET /allocation_candidates`` is enhanced to accept numbered groupings of -resource, required/forbidden trait, and aggregate association requests. A -``resources`` query parameter key with a positive integer suffix (e.g. -``resources42``) will be logically associated with ``required`` and/or -``member_of`` query parameter keys with the same suffix (e.g. ``required42``, -``member_of42``). The resources, required/forbidden traits, and aggregate -associations in that group will be satisfied by the same resource provider in -the response. When more than one numbered grouping is supplied, the -``group_policy`` query parameter is required to indicate how the groups should -interact. With ``group_policy=none``, separate groupings - numbered or -unnumbered - may or may not be satisfied by the same provider. With -``group_policy=isolate``, numbered groups are guaranteed to be satisfied by -*different* providers - though there may still be overlap with the unnumbered -group. In all cases, each ``allocation_request`` will be satisfied by providers -in a single non-sharing provider tree and/or sharing providers associated via -aggregate with any of the providers in that tree. - -The ``required`` and ``member_of`` query parameters for a given group are -optional. That is, you may specify ``resources42=XXX`` without a corresponding -``required42=YYY`` or ``member_of42=ZZZ``. However, the reverse (specifying -``required42=YYY`` or ``member_of42=ZZZ`` without ``resources42=XXX``) will -result in an error. - -The semantic of the (unnumbered) ``resources``, ``required``, and ``member_of`` -query parameters is unchanged: the resources, traits, and aggregate -associations specified thereby may be satisfied by any provider in the same -non-sharing tree or associated via the specified aggregate(s). - -1.26 Allow inventories to have reserved value equal to total ------------------------------------------------------------- - -.. versionadded:: Rocky - -Starting with this version, it is allowed to set the reserved value of the -resource provider inventory to be equal to total. - -1.27 Include all resource class inventories in provider_summaries ------------------------------------------------------------------ - -.. versionadded:: Rocky - -Include all resource class inventories in the ``provider_summaries`` field in -response of the ``GET /allocation_candidates`` API even if the resource class -is not in the requested resources. - -1.28 Consumer generation support --------------------------------- - -.. versionadded:: Rocky - -A new generation field has been added to the consumer concept. Consumers are -the actors that are allocated resources in the placement API. When an -allocation is created, a consumer UUID is specified. Starting with microversion -1.8, a project and user ID are also required. If using microversions prior to -1.8, these are populated from the ``incomplete_consumer_project_id`` and -``incomplete_consumer_user_id`` config options from the ``[placement]`` -section. - -The consumer generation facilitates safe concurrent modification of an -allocation. - -A consumer generation is now returned from the following URIs: - -``GET /resource_providers/{uuid}/allocations`` - -The response continues to be a dict with a key of ``allocations``, which itself -is a dict, keyed by consumer UUID, of allocations against the resource -provider. For each of those dicts, a ``consumer_generation`` field will now be -shown. - -``GET /allocations/{consumer_uuid}`` - -The response continues to be a dict with a key of ``allocations``, which -itself is a dict, keyed by resource provider UUID, of allocations being -consumed by the consumer with the ``{consumer_uuid}``. The top-level dict will -also now contain a ``consumer_generation`` field. - -The value of the ``consumer_generation`` field is opaque and should only be -used to send back to subsequent operations on the consumer's allocations. - -The ``PUT /allocations/{consumer_uuid}`` URI has been modified to now require a -``consumer_generation`` field in the request payload. This field is required to -be ``null`` if the caller expects that there are no allocations already -existing for the consumer. Otherwise, it should contain the generation that the -caller understands the consumer to be at the time of the call. - -A ``409 Conflict`` will be returned from ``PUT /allocations/{consumer_uuid}`` -if there was a mismatch between the supplied generation and the consumer's -generation as known by the server. Similarly, a ``409 Conflict`` will be -returned if during the course of replacing the consumer's allocations another -process concurrently changed the consumer's allocations. This allows the caller -to react to the concurrent write by re-reading the consumer's allocations and -re-issuing the call to replace allocations as needed. - -The ``PUT /allocations/{consumer_uuid}`` URI has also been modified to accept -an empty allocations object, thereby bringing it to parity with the behaviour -of ``POST /allocations``, which uses an empty allocations object to indicate -that the allocations for a particular consumer should be removed. Passing an -empty allocations object along with a ``consumer_generation`` makes ``PUT -/allocations/{consumer_uuid}`` a **safe** way to delete allocations for a -consumer. The ``DELETE /allocations/{consumer_uuid}`` URI remains unsafe to -call in deployments where multiple callers may simultaneously be attempting to -modify a consumer's allocations. - -The ``POST /allocations`` URI variant has also been changed to require a -``consumer_generation`` field in the request payload **for each consumer -involved in the request**. Similar responses to ``PUT -/allocations/{consumer_uuid}`` are returned when any of the consumers -generations conflict with the server's view of those consumers or if any of the -consumers involved in the request are modified by another process. - -.. warning:: In all cases, it is absolutely **NOT SAFE** to create and modify - allocations for a consumer using different microversions where one - of the microversions is prior to 1.28. The only way to safely - modify allocations for a consumer and satisfy expectations you - have regarding the prior existence (or lack of existence) of those - allocations is to always use microversion 1.28+ when calling - allocations API endpoints. - -1.29 Support allocation candidates with nested resource providers ------------------------------------------------------------------ - -.. versionadded:: Rocky - -Add support for nested resource providers with the following two features. -1) ``GET /allocation_candidates`` is aware of nested providers. Namely, when -provider trees are present, ``allocation_requests`` in the response of -``GET /allocation_candidates`` can include allocations on combinations of -multiple resource providers in the same tree. -2) ``root_provider_uuid`` and ``parent_provider_uuid`` are added to -``provider_summaries`` in the response of ``GET /allocation_candidates``. - -1.30 Provide a /reshaper resource ---------------------------------- - -Add support for a ``POST /reshaper`` resource that provides for atomically -migrating resource provider inventories and associated allocations when some of -the inventory moves from one resource provider to another, such as when a class -of inventory moves from a parent provider to a new child provider. - -.. note:: This is a special operation that should only be used in rare cases - of resource provider topology changing when inventory is in use. - Only use this if you are really sure of what you are doing. diff --git a/nova/api/openstack/placement/schemas/__init__.py b/nova/api/openstack/placement/schemas/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/api/openstack/placement/schemas/aggregate.py b/nova/api/openstack/placement/schemas/aggregate.py deleted file mode 100644 index dc5d94921665..000000000000 --- a/nova/api/openstack/placement/schemas/aggregate.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Aggregate schemas for Placement API.""" -import copy - - -_AGGREGATES_LIST_SCHEMA = { - "type": "array", - "items": { - "type": "string", - "format": "uuid" - }, - "uniqueItems": True -} - - -PUT_AGGREGATES_SCHEMA_V1_1 = copy.deepcopy(_AGGREGATES_LIST_SCHEMA) - - -PUT_AGGREGATES_SCHEMA_V1_19 = { - "type": "object", - "properties": { - "aggregates": copy.deepcopy(_AGGREGATES_LIST_SCHEMA), - "resource_provider_generation": { - "type": "integer", - } - }, - "required": [ - "aggregates", - "resource_provider_generation", - ], - "additionalProperties": False, -} diff --git a/nova/api/openstack/placement/schemas/allocation.py b/nova/api/openstack/placement/schemas/allocation.py deleted file mode 100644 index 169a953f58c3..000000000000 --- a/nova/api/openstack/placement/schemas/allocation.py +++ /dev/null @@ -1,169 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API schemas for setting and deleting allocations.""" - -import copy - -from nova.api.openstack.placement.schemas import common - - -ALLOCATION_SCHEMA = { - "type": "object", - "properties": { - "allocations": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "properties": { - "resource_provider": { - "type": "object", - "properties": { - "uuid": { - "type": "string", - "format": "uuid" - } - }, - "additionalProperties": False, - "required": ["uuid"] - }, - "resources": { - "type": "object", - "minProperties": 1, - "patternProperties": { - common.RC_PATTERN: { - "type": "integer", - "minimum": 1, - } - }, - "additionalProperties": False - } - }, - "required": [ - "resource_provider", - "resources" - ], - "additionalProperties": False - } - } - }, - "required": ["allocations"], - "additionalProperties": False -} - -ALLOCATION_SCHEMA_V1_8 = copy.deepcopy(ALLOCATION_SCHEMA) -ALLOCATION_SCHEMA_V1_8['properties']['project_id'] = {'type': 'string', - 'minLength': 1, - 'maxLength': 255} -ALLOCATION_SCHEMA_V1_8['properties']['user_id'] = {'type': 'string', - 'minLength': 1, - 'maxLength': 255} -ALLOCATION_SCHEMA_V1_8['required'].extend(['project_id', 'user_id']) - -# Update the allocation schema to achieve symmetry with the representation -# used when GET /allocations/{consumer_uuid} is called. -# NOTE(cdent): Explicit duplication here for sake of comprehensibility. -ALLOCATION_SCHEMA_V1_12 = { - "type": "object", - "properties": { - "allocations": { - "type": "object", - "minProperties": 1, - # resource provider uuid - "patternProperties": { - common.UUID_PATTERN: { - "type": "object", - "properties": { - # generation is optional - "generation": { - "type": "integer", - }, - "resources": { - "type": "object", - "minProperties": 1, - # resource class - "patternProperties": { - common.RC_PATTERN: { - "type": "integer", - "minimum": 1, - } - }, - "additionalProperties": False - } - }, - "required": ["resources"], - "additionalProperties": False - } - }, - "additionalProperties": False - }, - "project_id": { - "type": "string", - "minLength": 1, - "maxLength": 255 - }, - "user_id": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - "additionalProperties": False, - "required": [ - "allocations", - "project_id", - "user_id" - ] -} - - -# POST to /allocations, added in microversion 1.13, uses the -# POST_ALLOCATIONS_V1_13 schema to allow multiple allocations -# from multiple consumers in one request. It is a dict, keyed by -# consumer uuid, using the form of PUT allocations from microversion -# 1.12. In POST the allocations can be empty, so DELETABLE_ALLOCATIONS -# modifies ALLOCATION_SCHEMA_V1_12 accordingly. -DELETABLE_ALLOCATIONS = copy.deepcopy(ALLOCATION_SCHEMA_V1_12) -DELETABLE_ALLOCATIONS['properties']['allocations']['minProperties'] = 0 -POST_ALLOCATIONS_V1_13 = { - "type": "object", - "minProperties": 1, - "additionalProperties": False, - "patternProperties": { - common.UUID_PATTERN: DELETABLE_ALLOCATIONS - } -} - -# A required consumer generation was added to the top-level dict in this -# version of PUT /allocations/{consumer_uuid}. In addition, the PUT -# /allocations/{consumer_uuid}/now allows for empty allocations (indicating the -# allocations are being removed) -ALLOCATION_SCHEMA_V1_28 = copy.deepcopy(DELETABLE_ALLOCATIONS) -ALLOCATION_SCHEMA_V1_28['properties']['consumer_generation'] = { - "type": ["integer", "null"], - "additionalProperties": False -} -ALLOCATION_SCHEMA_V1_28['required'].append("consumer_generation") - -# A required consumer generation was added to the allocations dicts in this -# version of POST /allocations -REQUIRED_GENERATION_ALLOCS_POST = copy.deepcopy(DELETABLE_ALLOCATIONS) -alloc_props = REQUIRED_GENERATION_ALLOCS_POST['properties'] -alloc_props['consumer_generation'] = { - "type": ["integer", "null"], - "additionalProperties": False -} -REQUIRED_GENERATION_ALLOCS_POST['required'].append("consumer_generation") -POST_ALLOCATIONS_V1_28 = copy.deepcopy(POST_ALLOCATIONS_V1_13) -POST_ALLOCATIONS_V1_28["patternProperties"] = { - common.UUID_PATTERN: REQUIRED_GENERATION_ALLOCS_POST -} diff --git a/nova/api/openstack/placement/schemas/allocation_candidate.py b/nova/api/openstack/placement/schemas/allocation_candidate.py deleted file mode 100644 index d418366ff67c..000000000000 --- a/nova/api/openstack/placement/schemas/allocation_candidate.py +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API schemas for getting allocation candidates.""" - -import copy - - -# Represents the allowed query string parameters to the GET -# /allocation_candidates API call -GET_SCHEMA_1_10 = { - "type": "object", - "properties": { - "resources": { - "type": "string" - }, - }, - "required": [ - "resources", - ], - "additionalProperties": False, -} - - -# Add limit query parameter. -GET_SCHEMA_1_16 = copy.deepcopy(GET_SCHEMA_1_10) -GET_SCHEMA_1_16['properties']['limit'] = { - # A query parameter is always a string in webOb, but - # we'll handle integer here as well. - "type": ["integer", "string"], - "pattern": "^[1-9][0-9]*$", - "minimum": 1, - "minLength": 1 -} - -# Add required parameter. -GET_SCHEMA_1_17 = copy.deepcopy(GET_SCHEMA_1_16) -GET_SCHEMA_1_17['properties']['required'] = { - "type": ["string"] -} - -# Add member_of parameter. -GET_SCHEMA_1_21 = copy.deepcopy(GET_SCHEMA_1_17) -GET_SCHEMA_1_21['properties']['member_of'] = { - "type": ["string"] -} - -GET_SCHEMA_1_25 = copy.deepcopy(GET_SCHEMA_1_21) -# We're going to *replace* 'resources', 'required', and 'member_of'. -del GET_SCHEMA_1_25["properties"]["resources"] -del GET_SCHEMA_1_25["required"] -del GET_SCHEMA_1_25["properties"]["required"] -del GET_SCHEMA_1_25["properties"]["member_of"] -# Pattern property key format for a numbered or un-numbered grouping -_GROUP_PAT_FMT = "^%s([1-9][0-9]*)?$" -GET_SCHEMA_1_25["patternProperties"] = { - _GROUP_PAT_FMT % "resources": { - "type": "string", - }, - _GROUP_PAT_FMT % "required": { - "type": "string", - }, - _GROUP_PAT_FMT % "member_of": { - "type": "string", - }, -} -GET_SCHEMA_1_25["properties"]["group_policy"] = { - "type": "string", - "enum": ["none", "isolate"], -} diff --git a/nova/api/openstack/placement/schemas/common.py b/nova/api/openstack/placement/schemas/common.py deleted file mode 100644 index 51d3ee925cd5..000000000000 --- a/nova/api/openstack/placement/schemas/common.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -_UUID_CHAR = "[0-9a-fA-F-]" -# TODO(efried): Use this stricter pattern, and replace string/uuid with it: -# UUID_PATTERN = "^%s{8}-%s{4}-%s{4}-%s{4}-%s{12}$" % ((_UUID_CHAR,) * 5) -UUID_PATTERN = "^%s{36}$" % _UUID_CHAR - -_RC_TRAIT_CHAR = "[A-Z0-9_]" -_RC_TRAIT_PATTERN = "^%s+$" % _RC_TRAIT_CHAR -RC_PATTERN = _RC_TRAIT_PATTERN -_CUSTOM_RC_TRAIT_PATTERN = "^CUSTOM_%s+$" % _RC_TRAIT_CHAR -CUSTOM_RC_PATTERN = _CUSTOM_RC_TRAIT_PATTERN -CUSTOM_TRAIT_PATTERN = _CUSTOM_RC_TRAIT_PATTERN diff --git a/nova/api/openstack/placement/schemas/inventory.py b/nova/api/openstack/placement/schemas/inventory.py deleted file mode 100644 index cddea13064da..000000000000 --- a/nova/api/openstack/placement/schemas/inventory.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Inventory schemas for Placement API.""" - -import copy - -from nova.api.openstack.placement.schemas import common -from nova.db import constants as db_const - - -BASE_INVENTORY_SCHEMA = { - "type": "object", - "properties": { - "resource_provider_generation": { - "type": "integer" - }, - "total": { - "type": "integer", - "maximum": db_const.MAX_INT, - "minimum": 1, - }, - "reserved": { - "type": "integer", - "maximum": db_const.MAX_INT, - "minimum": 0, - }, - "min_unit": { - "type": "integer", - "maximum": db_const.MAX_INT, - "minimum": 1 - }, - "max_unit": { - "type": "integer", - "maximum": db_const.MAX_INT, - "minimum": 1 - }, - "step_size": { - "type": "integer", - "maximum": db_const.MAX_INT, - "minimum": 1 - }, - "allocation_ratio": { - "type": "number", - "maximum": db_const.SQL_SP_FLOAT_MAX - }, - }, - "required": [ - "total", - "resource_provider_generation" - ], - "additionalProperties": False -} - - -POST_INVENTORY_SCHEMA = copy.deepcopy(BASE_INVENTORY_SCHEMA) -POST_INVENTORY_SCHEMA['properties']['resource_class'] = { - "type": "string", - "pattern": common.RC_PATTERN, -} -POST_INVENTORY_SCHEMA['required'].append('resource_class') -POST_INVENTORY_SCHEMA['required'].remove('resource_provider_generation') - - -PUT_INVENTORY_RECORD_SCHEMA = copy.deepcopy(BASE_INVENTORY_SCHEMA) -PUT_INVENTORY_RECORD_SCHEMA['required'].remove('resource_provider_generation') -PUT_INVENTORY_SCHEMA = { - "type": "object", - "properties": { - "resource_provider_generation": { - "type": "integer" - }, - "inventories": { - "type": "object", - "patternProperties": { - common.RC_PATTERN: PUT_INVENTORY_RECORD_SCHEMA, - } - } - }, - "required": [ - "resource_provider_generation", - "inventories" - ], - "additionalProperties": False -} diff --git a/nova/api/openstack/placement/schemas/reshaper.py b/nova/api/openstack/placement/schemas/reshaper.py deleted file mode 100644 index 1658d925153e..000000000000 --- a/nova/api/openstack/placement/schemas/reshaper.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Reshaper schema for Placement API.""" - -import copy - -from nova.api.openstack.placement.schemas import allocation -from nova.api.openstack.placement.schemas import common -from nova.api.openstack.placement.schemas import inventory - - -ALLOCATIONS = copy.deepcopy(allocation.POST_ALLOCATIONS_V1_28) -# In the reshaper we need to allow allocations to be an empty dict -# because it may be the case that there simply are no allocations -# (now) for any of the inventory being moved. -ALLOCATIONS['minProperties'] = 0 -POST_RESHAPER_SCHEMA = { - "type": "object", - "properties": { - "inventories": { - "type": "object", - "patternProperties": { - # resource provider uuid - common.UUID_PATTERN: inventory.PUT_INVENTORY_SCHEMA, - }, - # We expect at least one inventories, otherwise there is no reason - # to call the reshaper. - "minProperties": 1, - "additionalProperties": False, - }, - "allocations": ALLOCATIONS, - }, - "required": [ - "inventories", - "allocations", - ], - "additionalProperties": False, -} diff --git a/nova/api/openstack/placement/schemas/resource_class.py b/nova/api/openstack/placement/schemas/resource_class.py deleted file mode 100644 index 32f75bc880d2..000000000000 --- a/nova/api/openstack/placement/schemas/resource_class.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API schemas for resource classes.""" - -import copy - -from nova.api.openstack.placement.schemas import common - - -POST_RC_SCHEMA_V1_2 = { - "type": "object", - "properties": { - "name": { - "type": "string", - "pattern": common.CUSTOM_RC_PATTERN, - "maxLength": 255, - }, - }, - "required": [ - "name" - ], - "additionalProperties": False, -} -PUT_RC_SCHEMA_V1_2 = copy.deepcopy(POST_RC_SCHEMA_V1_2) diff --git a/nova/api/openstack/placement/schemas/resource_provider.py b/nova/api/openstack/placement/schemas/resource_provider.py deleted file mode 100644 index 7ca43ef69ac9..000000000000 --- a/nova/api/openstack/placement/schemas/resource_provider.py +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API schemas for resource providers.""" - -import copy - - -POST_RESOURCE_PROVIDER_SCHEMA = { - "type": "object", - "properties": { - "name": { - "type": "string", - "maxLength": 200 - }, - "uuid": { - "type": "string", - "format": "uuid" - } - }, - "required": [ - "name" - ], - "additionalProperties": False, -} -# Remove uuid to create the schema for PUTting a resource provider -PUT_RESOURCE_PROVIDER_SCHEMA = copy.deepcopy(POST_RESOURCE_PROVIDER_SCHEMA) -PUT_RESOURCE_PROVIDER_SCHEMA['properties'].pop('uuid') - -# Placement API microversion 1.14 adds an optional parent_provider_uuid field -# to the POST and PUT request schemas -POST_RP_SCHEMA_V1_14 = copy.deepcopy(POST_RESOURCE_PROVIDER_SCHEMA) -POST_RP_SCHEMA_V1_14["properties"]["parent_provider_uuid"] = { - "anyOf": [ - { - "type": "string", - "format": "uuid", - }, - { - "type": "null", - } - ] -} -PUT_RP_SCHEMA_V1_14 = copy.deepcopy(POST_RP_SCHEMA_V1_14) -PUT_RP_SCHEMA_V1_14['properties'].pop('uuid') - -# Represents the allowed query string parameters to the GET /resource_providers -# API call -GET_RPS_SCHEMA_1_0 = { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "uuid": { - "type": "string", - "format": "uuid" - } - }, - "additionalProperties": False, -} - -# Placement API microversion 1.3 adds support for a member_of attribute -GET_RPS_SCHEMA_1_3 = copy.deepcopy(GET_RPS_SCHEMA_1_0) -GET_RPS_SCHEMA_1_3['properties']['member_of'] = { - "type": "string" -} - -# Placement API microversion 1.4 adds support for requesting resource providers -# having some set of capacity for some resources. The query string is a -# comma-delimited set of "$RESOURCE_CLASS_NAME:$AMOUNT" strings. The validation -# of the string is left up to the helper code in the -# normalize_resources_qs_param() function. -GET_RPS_SCHEMA_1_4 = copy.deepcopy(GET_RPS_SCHEMA_1_3) -GET_RPS_SCHEMA_1_4['properties']['resources'] = { - "type": "string" -} - -# Placement API microversion 1.14 adds support for requesting resource -# providers within a tree of providers. The 'in_tree' query string parameter -# should be the UUID of a resource provider. The result of the GET call will -# include only those resource providers in the same "provider tree" as the -# provider with the UUID represented by 'in_tree' -GET_RPS_SCHEMA_1_14 = copy.deepcopy(GET_RPS_SCHEMA_1_4) -GET_RPS_SCHEMA_1_14['properties']['in_tree'] = { - "type": "string", - "format": "uuid", -} - -# Microversion 1.18 adds support for the `required` query parameter to the -# `GET /resource_providers` API. It accepts a comma-separated list of string -# trait names. When specified, the API results will be filtered to include only -# resource providers marked with all the specified traits. This is in addition -# to (logical AND) any filtering based on other query parameters. -GET_RPS_SCHEMA_1_18 = copy.deepcopy(GET_RPS_SCHEMA_1_14) -GET_RPS_SCHEMA_1_18['properties']['required'] = { - "type": "string", -} diff --git a/nova/api/openstack/placement/schemas/trait.py b/nova/api/openstack/placement/schemas/trait.py deleted file mode 100644 index b9c04e54de2e..000000000000 --- a/nova/api/openstack/placement/schemas/trait.py +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Trait schemas for Placement API.""" - -import copy - -from nova.api.openstack.placement.schemas import common - - -TRAIT = { - "type": "string", - 'minLength': 1, 'maxLength': 255, -} - -CUSTOM_TRAIT = copy.deepcopy(TRAIT) -CUSTOM_TRAIT.update({"pattern": common.CUSTOM_TRAIT_PATTERN}) - -PUT_TRAITS_SCHEMA = { - "type": "object", - "properties": { - "traits": { - "type": "array", - "items": CUSTOM_TRAIT, - } - }, - 'required': ['traits'], - 'additionalProperties': False -} - -SET_TRAITS_FOR_RP_SCHEMA = copy.deepcopy(PUT_TRAITS_SCHEMA) -SET_TRAITS_FOR_RP_SCHEMA['properties']['traits']['items'] = TRAIT -SET_TRAITS_FOR_RP_SCHEMA['properties'][ - 'resource_provider_generation'] = {'type': 'integer'} -SET_TRAITS_FOR_RP_SCHEMA['required'].append('resource_provider_generation') - -LIST_TRAIT_SCHEMA = { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "associated": { - "type": "string", - } - }, - "additionalProperties": False -} diff --git a/nova/api/openstack/placement/schemas/usage.py b/nova/api/openstack/placement/schemas/usage.py deleted file mode 100644 index 3b1a1845046d..000000000000 --- a/nova/api/openstack/placement/schemas/usage.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Placement API schemas for usage information.""" - -# Represents the allowed query string parameters to GET /usages -GET_USAGES_SCHEMA_1_9 = { - "type": "object", - "properties": { - "project_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - }, - "user_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - }, - }, - "required": [ - "project_id" - ], - "additionalProperties": False, -} diff --git a/nova/api/openstack/placement/util.py b/nova/api/openstack/placement/util.py deleted file mode 100644 index 6b3ae052f63c..000000000000 --- a/nova/api/openstack/placement/util.py +++ /dev/null @@ -1,697 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Utility methods for placement API.""" - -import functools -import re - -import jsonschema -from oslo_config import cfg -from oslo_log import log as logging -from oslo_middleware import request_id -from oslo_serialization import jsonutils -from oslo_utils import timeutils -from oslo_utils import uuidutils -import webob - -from nova.api.openstack.placement import errors -from nova.api.openstack.placement import exception -from nova.api.openstack.placement import lib as placement_lib -# NOTE(cdent): avoid cyclical import conflict between util and -# microversion -import nova.api.openstack.placement.microversion -from nova.api.openstack.placement.objects import consumer as consumer_obj -from nova.api.openstack.placement.objects import project as project_obj -from nova.api.openstack.placement.objects import user as user_obj -from nova.i18n import _ - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - -# Error code handling constants -ENV_ERROR_CODE = 'placement.error_code' -ERROR_CODE_MICROVERSION = (1, 23) - -# Querystring-related constants -_QS_RESOURCES = 'resources' -_QS_REQUIRED = 'required' -_QS_MEMBER_OF = 'member_of' -_QS_KEY_PATTERN = re.compile( - r"^(%s)([1-9][0-9]*)?$" % '|'.join( - (_QS_RESOURCES, _QS_REQUIRED, _QS_MEMBER_OF))) - - -# NOTE(cdent): This registers a FormatChecker on the jsonschema -# module. Do not delete this code! Although it appears that nothing -# is using the decorated method it is being used in JSON schema -# validations to check uuid formatted strings. -@jsonschema.FormatChecker.cls_checks('uuid') -def _validate_uuid_format(instance): - return uuidutils.is_uuid_like(instance) - - -def check_accept(*types): - """If accept is set explicitly, try to follow it. - - If there is no match for the incoming accept header - send a 406 response code. - - If accept is not set send our usual content-type in - response. - """ - def decorator(f): - @functools.wraps(f) - def decorated_function(req): - if req.accept: - best_matches = req.accept.acceptable_offers(types) - if not best_matches: - type_string = ', '.join(types) - raise webob.exc.HTTPNotAcceptable( - _('Only %(type)s is provided') % {'type': type_string}, - json_formatter=json_error_formatter) - return f(req) - return decorated_function - return decorator - - -def extract_json(body, schema): - """Extract JSON from a body and validate with the provided schema.""" - try: - data = jsonutils.loads(body) - except ValueError as exc: - raise webob.exc.HTTPBadRequest( - _('Malformed JSON: %(error)s') % {'error': exc}, - json_formatter=json_error_formatter) - try: - jsonschema.validate(data, schema, - format_checker=jsonschema.FormatChecker()) - except jsonschema.ValidationError as exc: - raise webob.exc.HTTPBadRequest( - _('JSON does not validate: %(error)s') % {'error': exc}, - json_formatter=json_error_formatter) - return data - - -def inventory_url(environ, resource_provider, resource_class=None): - url = '%s/inventories' % resource_provider_url(environ, resource_provider) - if resource_class: - url = '%s/%s' % (url, resource_class) - return url - - -def json_error_formatter(body, status, title, environ): - """A json_formatter for webob exceptions. - - Follows API-WG guidelines at - http://specs.openstack.org/openstack/api-wg/guidelines/errors.html - """ - # Shortcut to microversion module, to avoid wraps below. - microversion = nova.api.openstack.placement.microversion - - # Clear out the html that webob sneaks in. - body = webob.exc.strip_tags(body) - # Get status code out of status message. webob's error formatter - # only passes entire status string. - status_code = int(status.split(None, 1)[0]) - error_dict = { - 'status': status_code, - 'title': title, - 'detail': body - } - - # Version may not be set if we have experienced an error before it - # is set. - want_version = environ.get(microversion.MICROVERSION_ENVIRON) - if want_version and want_version.matches(ERROR_CODE_MICROVERSION): - error_dict['code'] = environ.get(ENV_ERROR_CODE, errors.DEFAULT) - - # If the request id middleware has had a chance to add an id, - # put it in the error response. - if request_id.ENV_REQUEST_ID in environ: - error_dict['request_id'] = environ[request_id.ENV_REQUEST_ID] - - # When there is a no microversion in the environment and a 406, - # microversion parsing failed so we need to include microversion - # min and max information in the error response. - if status_code == 406 and microversion.MICROVERSION_ENVIRON not in environ: - error_dict['max_version'] = microversion.max_version_string() - error_dict['min_version'] = microversion.min_version_string() - - return {'errors': [error_dict]} - - -def pick_last_modified(last_modified, obj): - """Choose max of last_modified and obj.updated_at or obj.created_at. - - If updated_at is not implemented in `obj` use the current time in UTC. - """ - try: - current_modified = (obj.updated_at or obj.created_at) - except NotImplementedError: - # If updated_at is not implemented, we are looking at objects that - # have not come from the database, so "now" is the right modified - # time. - current_modified = timeutils.utcnow(with_timezone=True) - if last_modified: - last_modified = max(last_modified, current_modified) - else: - last_modified = current_modified - return last_modified - - -def require_content(content_type): - """Decorator to require a content type in a handler.""" - def decorator(f): - @functools.wraps(f) - def decorated_function(req): - if req.content_type != content_type: - # webob's unset content_type is the empty string so - # set it the error message content to 'None' to make - # a useful message in that case. This also avoids a - # KeyError raised when webob.exc eagerly fills in a - # Template for output we will never use. - if not req.content_type: - req.content_type = 'None' - raise webob.exc.HTTPUnsupportedMediaType( - _('The media type %(bad_type)s is not supported, ' - 'use %(good_type)s') % - {'bad_type': req.content_type, - 'good_type': content_type}, - json_formatter=json_error_formatter) - else: - return f(req) - return decorated_function - return decorator - - -def resource_class_url(environ, resource_class): - """Produce the URL for a resource class. - - If SCRIPT_NAME is present, it is the mount point of the placement - WSGI app. - """ - prefix = environ.get('SCRIPT_NAME', '') - return '%s/resource_classes/%s' % (prefix, resource_class.name) - - -def resource_provider_url(environ, resource_provider): - """Produce the URL for a resource provider. - - If SCRIPT_NAME is present, it is the mount point of the placement - WSGI app. - """ - prefix = environ.get('SCRIPT_NAME', '') - return '%s/resource_providers/%s' % (prefix, resource_provider.uuid) - - -def trait_url(environ, trait): - """Produce the URL for a trait. - - If SCRIPT_NAME is present, it is the mount point of the placement - WSGI app. - """ - prefix = environ.get('SCRIPT_NAME', '') - return '%s/traits/%s' % (prefix, trait.name) - - -def validate_query_params(req, schema): - try: - # NOTE(Kevin_Zheng): The webob package throws UnicodeError when - # param cannot be decoded. Catch this and raise HTTP 400. - jsonschema.validate(dict(req.GET), schema, - format_checker=jsonschema.FormatChecker()) - except (jsonschema.ValidationError, UnicodeDecodeError) as exc: - raise webob.exc.HTTPBadRequest( - _('Invalid query string parameters: %(exc)s') % - {'exc': exc}) - - -def wsgi_path_item(environ, name): - """Extract the value of a named field in a URL. - - Return None if the name is not present or there are no path items. - """ - # NOTE(cdent): For the time being we don't need to urldecode - # the value as the entire placement API has paths that accept no - # encoded values. - try: - return environ['wsgiorg.routing_args'][1][name] - except (KeyError, IndexError): - return None - - -def normalize_resources_qs_param(qs): - """Given a query string parameter for resources, validate it meets the - expected format and return a dict of amounts, keyed by resource class name. - - The expected format of the resources parameter looks like so: - - $RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT - - So, if the user was looking for resource providers that had room for an - instance that will consume 2 vCPUs, 1024 MB of RAM and 50GB of disk space, - they would use the following query string: - - ?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50 - - The returned value would be: - - { - "VCPU": 2, - "MEMORY_MB": 1024, - "DISK_GB": 50, - } - - :param qs: The value of the 'resources' query string parameter - :raises `webob.exc.HTTPBadRequest` if the parameter's value isn't in the - expected format. - """ - if qs.strip() == "": - msg = _('Badly formed resources parameter. Expected resources ' - 'query string parameter in form: ' - '?resources=VCPU:2,MEMORY_MB:1024. Got: empty string.') - raise webob.exc.HTTPBadRequest(msg) - - result = {} - resource_tuples = qs.split(',') - for rt in resource_tuples: - try: - rc_name, amount = rt.split(':') - except ValueError: - msg = _('Badly formed resources parameter. Expected resources ' - 'query string parameter in form: ' - '?resources=VCPU:2,MEMORY_MB:1024. Got: %s.') - msg = msg % rt - raise webob.exc.HTTPBadRequest(msg) - try: - amount = int(amount) - except ValueError: - msg = _('Requested resource %(resource_name)s expected positive ' - 'integer amount. Got: %(amount)s.') - msg = msg % { - 'resource_name': rc_name, - 'amount': amount, - } - raise webob.exc.HTTPBadRequest(msg) - if amount < 1: - msg = _('Requested resource %(resource_name)s requires ' - 'amount >= 1. Got: %(amount)d.') - msg = msg % { - 'resource_name': rc_name, - 'amount': amount, - } - raise webob.exc.HTTPBadRequest(msg) - result[rc_name] = amount - return result - - -def valid_trait(trait, allow_forbidden): - """Return True if the provided trait is the expected form. - - When allow_forbidden is True, then a leading '!' is acceptable. - """ - if trait.startswith('!') and not allow_forbidden: - return False - return True - - -def normalize_traits_qs_param(val, allow_forbidden=False): - """Parse a traits query string parameter value. - - Note that this method doesn't know or care about the query parameter key, - which may currently be of the form `required`, `required123`, etc., but - which may someday also include `preferred`, etc. - - This method currently does no format validation of trait strings, other - than to ensure they're not zero-length. - - :param val: A traits query parameter value: a comma-separated string of - trait names. - :param allow_forbidden: If True, accept forbidden traits (that is, traits - prefixed by '!') as a valid form when notifying - the caller that the provided value is not properly - formed. - :return: A set of trait names. - :raises `webob.exc.HTTPBadRequest` if the val parameter is not in the - expected format. - """ - ret = set(substr.strip() for substr in val.split(',')) - expected_form = 'HW_CPU_X86_VMX,CUSTOM_MAGIC' - if allow_forbidden: - expected_form = 'HW_CPU_X86_VMX,!CUSTOM_MAGIC' - if not all(trait and valid_trait(trait, allow_forbidden) for trait in ret): - msg = _("Invalid query string parameters: Expected 'required' " - "parameter value of the form: %(form)s. " - "Got: %(val)s") % {'form': expected_form, 'val': val} - raise webob.exc.HTTPBadRequest(msg) - return ret - - -def normalize_member_of_qs_params(req, suffix=''): - """Given a webob.Request object, validate that the member_of querystring - parameters are correct. We begin supporting multiple member_of params in - microversion 1.24. - - :param req: webob.Request object - :return: A list containing sets of UUIDs of aggregates to filter on - :raises `webob.exc.HTTPBadRequest` if the microversion requested is <1.24 - and the request contains multiple member_of querystring params - :raises `webob.exc.HTTPBadRequest` if the val parameter is not in the - expected format. - """ - microversion = nova.api.openstack.placement.microversion - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - multi_member_of = want_version.matches((1, 24)) - if not multi_member_of and len(req.GET.getall('member_of' + suffix)) > 1: - raise webob.exc.HTTPBadRequest( - _('Multiple member_of%s parameters are not supported') % suffix) - values = [] - for value in req.GET.getall('member_of' + suffix): - values.append(normalize_member_of_qs_param(value)) - return values - - -def normalize_member_of_qs_param(value): - """Parse a member_of query string parameter value. - - Valid values are either a single UUID, or the prefix 'in:' followed by two - or more comma-separated UUIDs. - - :param value: A member_of query parameter of either a single UUID, or a - comma-separated string of two or more UUIDs, prefixed with - the "in:" operator - :return: A set of UUIDs - :raises `webob.exc.HTTPBadRequest` if the value parameter is not in the - expected format. - """ - if "," in value and not value.startswith("in:"): - msg = _("Multiple values for 'member_of' must be prefixed with the " - "'in:' keyword. Got: %s") % value - raise webob.exc.HTTPBadRequest(msg) - if value.startswith('in:'): - value = set(value[3:].split(',')) - else: - value = set([value]) - # Make sure the values are actually UUIDs. - for aggr_uuid in value: - if not uuidutils.is_uuid_like(aggr_uuid): - msg = _("Invalid query string parameters: Expected 'member_of' " - "parameter to contain valid UUID(s). Got: %s") % aggr_uuid - raise webob.exc.HTTPBadRequest(msg) - return value - - -def parse_qs_request_groups(req): - """Parse numbered resources, traits, and member_of groupings out of a - querystring dict. - - The input qsdict represents a query string of the form: - - ?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT - &required=$TRAIT_NAME,$TRAIT_NAME&member_of=in:$AGG1_UUID,$AGG2_UUID - &resources1=$RESOURCE_CLASS_NAME:$AMOUNT,RESOURCE_CLASS_NAME:$AMOUNT - &required1=$TRAIT_NAME,$TRAIT_NAME&member_of1=$AGG_UUID - &resources2=$RESOURCE_CLASS_NAME:$AMOUNT,RESOURCE_CLASS_NAME:$AMOUNT - &required2=$TRAIT_NAME,$TRAIT_NAME&member_of2=$AGG_UUID - - These are parsed in groups according to the numeric suffix of the key. - For each group, a RequestGroup instance is created containing that group's - resources, required traits, and member_of. For the (single) group with no - suffix, the RequestGroup.use_same_provider attribute is False; for the - numbered groups it is True. - - If a trait in the required parameter is prefixed with ``!`` this - indicates that that trait must not be present on the resource - providers in the group. That is, the trait is forbidden. Forbidden traits - are only processed if ``allow_forbidden`` is True. This allows the - caller to control processing based on microversion handling. - - The return is a dict, keyed by the numeric suffix of these RequestGroup - instances (or the empty string for the unnumbered group). - - As an example, if qsdict represents the query string: - - ?resources=VCPU:2,MEMORY_MB:1024,DISK_GB=50 - &required=HW_CPU_X86_VMX,CUSTOM_STORAGE_RAID - &member_of=in:9323b2b1-82c9-4e91-bdff-e95e808ef954,8592a199-7d73-4465-8df6-ab00a6243c82 # noqa - &resources1=SRIOV_NET_VF:2 - &required1=CUSTOM_PHYSNET_PUBLIC,CUSTOM_SWITCH_A - &resources2=SRIOV_NET_VF:1 - &required2=!CUSTOM_PHYSNET_PUBLIC - - ...the return value will be: - - { '': RequestGroup( - use_same_provider=False, - resources={ - "VCPU": 2, - "MEMORY_MB": 1024, - "DISK_GB" 50, - }, - required_traits=[ - "HW_CPU_X86_VMX", - "CUSTOM_STORAGE_RAID", - ], - member_of=[ - [9323b2b1-82c9-4e91-bdff-e95e808ef954], - [8592a199-7d73-4465-8df6-ab00a6243c82, - ddbd9226-d6a6-475e-a85f-0609914dd058], - ], - ), - '1': RequestGroup( - use_same_provider=True, - resources={ - "SRIOV_NET_VF": 2, - }, - required_traits=[ - "CUSTOM_PHYSNET_PUBLIC", - "CUSTOM_SWITCH_A", - ], - ), - '2': RequestGroup( - use_same_provider=True, - resources={ - "SRIOV_NET_VF": 1, - }, - forbidden_traits=[ - "CUSTOM_PHYSNET_PUBLIC", - ], - ), - } - - :param req: webob.Request object - :return: A list of RequestGroup instances. - :raises `webob.exc.HTTPBadRequest` if any value is malformed, or if a - trait list is given without corresponding resources. - """ - microversion = nova.api.openstack.placement.microversion - want_version = req.environ[microversion.MICROVERSION_ENVIRON] - # Control whether we handle forbidden traits. - allow_forbidden = want_version.matches((1, 22)) - # Temporary dict of the form: { suffix: RequestGroup } - by_suffix = {} - - def get_request_group(suffix): - if suffix not in by_suffix: - rq_grp = placement_lib.RequestGroup(use_same_provider=bool(suffix)) - by_suffix[suffix] = rq_grp - return by_suffix[suffix] - - for key, val in req.GET.items(): - match = _QS_KEY_PATTERN.match(key) - if not match: - continue - # `prefix` is 'resources', 'required', or 'member_of' - # `suffix` is an integer string, or None - prefix, suffix = match.groups() - suffix = suffix or '' - request_group = get_request_group(suffix) - if prefix == _QS_RESOURCES: - request_group.resources = normalize_resources_qs_param(val) - elif prefix == _QS_REQUIRED: - request_group.required_traits = normalize_traits_qs_param( - val, allow_forbidden=allow_forbidden) - elif prefix == _QS_MEMBER_OF: - # special handling of member_of qparam since we allow multiple - # member_of params at microversion 1.24. - # NOTE(jaypipes): Yes, this is inefficient to do this when there - # are multiple member_of query parameters, but we do this so we can - # error out if someone passes an "orphaned" member_of request - # group. - # TODO(jaypipes): Do validation of query parameters using - # JSONSchema - request_group.member_of = normalize_member_of_qs_params( - req, suffix) - - # Ensure any group with 'required' or 'member_of' also has 'resources'. - orphans = [('required%s' % suff) for suff, group in by_suffix.items() - if group.required_traits and not group.resources] - if orphans: - msg = _('All traits parameters must be associated with resources. ' - 'Found the following orphaned traits keys: %s') - raise webob.exc.HTTPBadRequest(msg % ', '.join(orphans)) - orphans = [('member_of%s' % suff) for suff, group in by_suffix.items() - if group.member_of and not group.resources] - if orphans: - msg = _('All member_of parameters must be associated with ' - 'resources. Found the following orphaned member_of ' - 'keys: %s') - raise webob.exc.HTTPBadRequest(msg % ', '.join(orphans)) - # All request groups must have resources (which is almost, but not quite, - # verified by the orphan checks above). - if not all(grp.resources for grp in by_suffix.values()): - msg = _("All request groups must specify resources.") - raise webob.exc.HTTPBadRequest(msg) - # The above would still pass if there were no request groups - if not by_suffix: - msg = _("At least one request group (`resources` or `resources{N}`) " - "is required.") - raise webob.exc.HTTPBadRequest(msg) - - # Make adjustments for forbidden traits by stripping forbidden out - # of required. - if allow_forbidden: - conflicting_traits = [] - for suff, group in by_suffix.items(): - forbidden = [trait for trait in group.required_traits - if trait.startswith('!')] - group.required_traits = (group.required_traits - set(forbidden)) - group.forbidden_traits = set([trait.lstrip('!') for trait in - forbidden]) - conflicts = group.forbidden_traits & group.required_traits - if conflicts: - conflicting_traits.append('required%s: (%s)' - % (suff, ', '.join(conflicts))) - if conflicting_traits: - msg = _('Conflicting required and forbidden traits found in the ' - 'following traits keys: %s') - raise webob.exc.HTTPBadRequest(msg % ', '.join(conflicting_traits)) - - return by_suffix - - -def ensure_consumer(ctx, consumer_uuid, project_id, user_id, - consumer_generation, want_version): - """Ensures there are records in the consumers, projects and users table for - the supplied external identifiers. - - Returns a tuple containing the populated Consumer object containing Project - and User sub-objects and a boolean indicating whether a new Consumer object - was created (as opposed to an existing consumer record retrieved) - - :note: If the supplied project or user external identifiers do not match an - existing consumer's project and user identifiers, the existing - consumer's project and user IDs are updated to reflect the supplied - ones. - - :param ctx: The request context. - :param consumer_uuid: The uuid of the consumer of the resources. - :param project_id: The external ID of the project consuming the resources. - :param user_id: The external ID of the user consuming the resources. - :param consumer_generation: The generation provided by the user for this - consumer. - :param want_version: the microversion matcher. - :raises webob.exc.HTTPConflict if consumer generation is required and there - was a mismatch - """ - created_new_consumer = False - requires_consumer_generation = want_version.matches((1, 28)) - if project_id is None: - project_id = CONF.placement.incomplete_consumer_project_id - user_id = CONF.placement.incomplete_consumer_user_id - try: - proj = project_obj.Project.get_by_external_id(ctx, project_id) - except exception.NotFound: - # Auto-create the project if we found no record of it... - try: - proj = project_obj.Project(ctx, external_id=project_id) - proj.create() - except exception.ProjectExists: - # No worries, another thread created this project already - proj = project_obj.Project.get_by_external_id(ctx, project_id) - try: - user = user_obj.User.get_by_external_id(ctx, user_id) - except exception.NotFound: - # Auto-create the user if we found no record of it... - try: - user = user_obj.User(ctx, external_id=user_id) - user.create() - except exception.UserExists: - # No worries, another thread created this user already - user = user_obj.User.get_by_external_id(ctx, user_id) - - try: - consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) - if requires_consumer_generation: - if consumer.generation != consumer_generation: - raise webob.exc.HTTPConflict( - _('consumer generation conflict - ' - 'expected %(expected_gen)s but got %(got_gen)s') % - { - 'expected_gen': consumer.generation, - 'got_gen': consumer_generation, - }, - comment=errors.CONCURRENT_UPDATE) - # NOTE(jaypipes): The user may have specified a different project and - # user external ID than the one that we had for the consumer. If this - # is the case, go ahead and modify the consumer record with the - # newly-supplied project/user information, but do not bump the consumer - # generation (since it will be bumped in the - # AllocationList.replace_all() method). - # - # TODO(jaypipes): This means that there may be a partial update. - # Imagine a scenario where a user calls POST /allocations, and the - # payload references two consumers. The first consumer is a new - # consumer and is auto-created. The second consumer is an existing - # consumer, but contains a different project or user ID than the - # existing consumer's record. If the eventual call to - # AllocationList.replace_all() fails for whatever reason (say, a - # resource provider generation conflict or out of resources failure), - # we will end up deleting the auto-created consumer but we MAY not undo - # the changes to the second consumer's project and user ID. I say MAY - # and not WILL NOT because I'm not sure that the exception that gets - # raised from AllocationList.replace_all() will cause the context - # manager's transaction to rollback automatically. I believe that the - # same transaction context is used for both util.ensure_consumer() and - # AllocationList.replace_all() within the same HTTP request, but need - # to test this to be 100% certain... - if (project_id != consumer.project.external_id or - user_id != consumer.user.external_id): - LOG.debug("Supplied project or user ID for consumer %s was " - "different than existing record. Updating consumer " - "record.", consumer_uuid) - consumer.project = proj - consumer.user = user - consumer.update() - except exception.NotFound: - # If we are attempting to modify or create allocations after 1.26, we - # need a consumer generation specified. The user must have specified - # None for the consumer generation if we get here, since there was no - # existing consumer with this UUID and therefore the user should be - # indicating that they expect the consumer did not exist. - if requires_consumer_generation: - if consumer_generation is not None: - raise webob.exc.HTTPConflict( - _('consumer generation conflict - ' - 'expected null but got %s') % consumer_generation, - comment=errors.CONCURRENT_UPDATE) - # No such consumer. This is common for new allocations. Create the - # consumer record - try: - consumer = consumer_obj.Consumer( - ctx, uuid=consumer_uuid, project=proj, user=user) - consumer.create() - created_new_consumer = True - except exception.ConsumerExists: - # No worries, another thread created this user already - consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) - return consumer, created_new_consumer diff --git a/nova/api/openstack/placement/wsgi.py b/nova/api/openstack/placement/wsgi.py deleted file mode 100644 index a37e5954ebf2..000000000000 --- a/nova/api/openstack/placement/wsgi.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""WSGI script for Placement API - -WSGI handler for running Placement API under Apache2, nginx, gunicorn etc. -""" - -import logging as py_logging -import os -import os.path - -from oslo_log import log as logging -from oslo_middleware import cors -from oslo_policy import opts as policy_opts -from oslo_utils import importutils -import pbr.version - -from nova.api.openstack.placement import db_api -from nova.api.openstack.placement import deploy -from nova import conf - - -profiler = importutils.try_import('osprofiler.opts') - - -CONFIG_FILE = 'nova.conf' - - -version_info = pbr.version.VersionInfo('nova') - - -def setup_logging(config): - # Any dependent libraries that have unhelp debug levels should be - # pinned to a higher default. - extra_log_level_defaults = [ - 'routes=INFO', - ] - logging.set_defaults(default_log_levels=logging.get_default_log_levels() + - extra_log_level_defaults) - logging.setup(config, 'nova') - py_logging.captureWarnings(True) - - -def _get_config_file(env=None): - if env is None: - env = os.environ - - dirname = env.get('OS_PLACEMENT_CONFIG_DIR', '/etc/nova').strip() - return os.path.join(dirname, CONFIG_FILE) - - -def _parse_args(argv, default_config_files): - logging.register_options(conf.CONF) - - if profiler: - profiler.set_defaults(conf.CONF) - - _set_middleware_defaults() - - # This is needed so we can check [oslo_policy]/enforce_scope in the - # deploy module. - policy_opts.set_defaults(conf.CONF) - - conf.CONF(argv[1:], project='nova', version=version_info.version_string(), - default_config_files=default_config_files) - - -def _set_middleware_defaults(): - """Update default configuration options for oslo.middleware.""" - cors.set_defaults( - allow_headers=['X-Auth-Token', - 'X-Openstack-Request-Id', - 'X-Identity-Status', - 'X-Roles', - 'X-Service-Catalog', - 'X-User-Id', - 'X-Tenant-Id'], - expose_headers=['X-Auth-Token', - 'X-Openstack-Request-Id', - 'X-Subject-Token', - 'X-Service-Token'], - allow_methods=['GET', - 'PUT', - 'POST', - 'DELETE', - 'PATCH'] - ) - - -def init_application(): - # initialize the config system - conffile = _get_config_file() - - # NOTE(lyarwood): Call reset to ensure the ConfigOpts object doesn't - # already contain registered options if the app is reloaded. - conf.CONF.reset() - - _parse_args([], default_config_files=[conffile]) - db_api.configure(conf.CONF) - - # initialize the logging system - setup_logging(conf.CONF) - - # dump conf at debug if log_options - if conf.CONF.log_options: - conf.CONF.log_opt_values( - logging.getLogger(__name__), - logging.DEBUG) - - # build and return our WSGI app - return deploy.loadapp(conf.CONF) diff --git a/nova/api/openstack/placement/wsgi_wrapper.py b/nova/api/openstack/placement/wsgi_wrapper.py deleted file mode 100644 index fcb6551d3e4f..000000000000 --- a/nova/api/openstack/placement/wsgi_wrapper.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Extend functionality from webob.dec.wsgify for Placement API.""" - -import webob - -from oslo_log import log as logging -from webob.dec import wsgify - -from nova.api.openstack.placement import util - -LOG = logging.getLogger(__name__) - - -class PlacementWsgify(wsgify): - - def call_func(self, req, *args, **kwargs): - """Add json_error_formatter to any webob HTTPExceptions.""" - try: - super(PlacementWsgify, self).call_func(req, *args, **kwargs) - except webob.exc.HTTPException as exc: - LOG.debug("Placement API returning an error response: %s", exc) - exc.json_formatter = util.json_error_formatter - # The exception itself is not passed to json_error_formatter - # but environ is, so set the environ. - if exc.comment: - req.environ[util.ENV_ERROR_CODE] = exc.comment - exc.comment = None - raise diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index a2cebb8e1efd..a060d77f9576 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -45,8 +45,6 @@ import six import six.moves.urllib.parse as urlparse from sqlalchemy.engine import url as sqla_url -# FIXME(cdent): This is a speedbump in the extraction process -from nova.api.openstack.placement.objects import consumer as consumer_obj from nova.cmd import common as cmd_common from nova.compute import api as compute_api import nova.conf @@ -400,9 +398,6 @@ class DbCommands(object): # need to be populated if it was not specified during boot time. instance_obj.populate_missing_availability_zones, # Added in Rocky - # FIXME(cdent): This is a factor that needs to be addressed somehow - consumer_obj.create_incomplete_consumers, - # Added in Rocky instance_mapping_obj.populate_queued_for_delete, # Added in Stein compute_node_obj.migrate_empty_ratio, @@ -864,11 +859,7 @@ class ApiDbCommands(object): @args('--version', metavar='', help=argparse.SUPPRESS) @args('version2', metavar='VERSION', nargs='?', help='Database version') def sync(self, version=None, version2=None): - """Sync the database up to the most recent version. - - If placement_database.connection is not None, sync that - database using the API database migrations. - """ + """Sync the database up to the most recent version.""" if version and not version2: print(_("DEPRECATED: The '--version' parameter was deprecated in " "the Pike cycle and will not be supported in future " @@ -876,15 +867,7 @@ class ApiDbCommands(object): "instead")) version2 = version - # NOTE(cdent): At the moment, the migration code deep in the belly - # of the migration package doesn't actually return anything, so - # returning the result of db_sync is not particularly meaningful - # here. But, in case that changes, we store the result from the - # the placement sync to and with the api sync. - result = True - if CONF.placement_database.connection is not None: - result = migration.db_sync(version2, database='placement') - return migration.db_sync(version2, database='api') and result + return migration.db_sync(version2, database='api') def version(self): """Print the current database version.""" @@ -1844,7 +1827,6 @@ class PlacementCommands(object): return num_processed - # FIXME(cdent): This needs to be addressed as part of extraction. @action_description( _("Iterates over non-cell0 cells looking for instances which do " "not have allocations in the Placement service, or have incomplete " diff --git a/nova/conf/database.py b/nova/conf/database.py index 253f418159ee..90e88b81cb21 100644 --- a/nova/conf/database.py +++ b/nova/conf/database.py @@ -106,61 +106,9 @@ def enrich_help_text(alt_db_opts): alt_db_opt.help = db_opt.help + alt_db_opt.help -# NOTE(cdent): See the note above on api_db_group. The same issues -# apply here. - -placement_db_group = cfg.OptGroup('placement_database', - title='Placement API database options', - help=""" -The *Placement API Database* is a separate database which can be used with the -placement service. This database is optional: if the connection option is not -set, the nova api database will be used instead. -""") - -placement_db_opts = [ - cfg.StrOpt('connection', - help='', - secret=True), - cfg.StrOpt('connection_parameters', - default='', - help=''), - cfg.BoolOpt('sqlite_synchronous', - default=True, - help=''), - cfg.StrOpt('slave_connection', - secret=True, - help=''), - cfg.StrOpt('mysql_sql_mode', - default='TRADITIONAL', - help=''), - cfg.IntOpt('connection_recycle_time', - default=3600, - help=''), - cfg.IntOpt('max_pool_size', - help=''), - cfg.IntOpt('max_retries', - default=10, - help=''), - cfg.IntOpt('retry_interval', - default=10, - help=''), - cfg.IntOpt('max_overflow', - help=''), - cfg.IntOpt('connection_debug', - default=0, - help=''), - cfg.BoolOpt('connection_trace', - default=False, - help=''), - cfg.IntOpt('pool_timeout', - help=''), -] # noqa - - def register_opts(conf): oslo_db_options.set_defaults(conf, connection=_DEFAULT_SQL_CONNECTION) conf.register_opts(api_db_opts, group=api_db_group) - conf.register_opts(placement_db_opts, group=placement_db_group) def list_opts(): @@ -174,9 +122,7 @@ def list_opts(): global _ENRICHED if not _ENRICHED: enrich_help_text(api_db_opts) - enrich_help_text(placement_db_opts) _ENRICHED = True return { api_db_group: api_db_opts, - placement_db_group: placement_db_opts, } diff --git a/nova/conf/placement.py b/nova/conf/placement.py index 31eb3b403a8d..625c48f07004 100644 --- a/nova/conf/placement.py +++ b/nova/conf/placement.py @@ -17,80 +17,22 @@ from nova.conf import utils as confutils DEFAULT_SERVICE_TYPE = 'placement' -DEFAULT_CONSUMER_MISSING_ID = '00000000-0000-0000-0000-000000000000' + placement_group = cfg.OptGroup( 'placement', title='Placement Service Options', help="Configuration options for connecting to the placement API service") -placement_opts = [ - cfg.BoolOpt( - 'randomize_allocation_candidates', - default=False, - help=""" -If True, when limiting allocation candidate results, the results will be -a random sampling of the full result set. If False, allocation candidates -are returned in a deterministic but undefined order. That is, all things -being equal, two requests for allocation candidates will return the same -results in the same order; but no guarantees are made as to how that order -is determined. -"""), - # TODO(mriedem): When placement is split out of nova, this should be - # deprecated since then [oslo_policy]/policy_file can be used. - cfg.StrOpt( - 'policy_file', - # This default matches what is in - # etc/nova/placement-policy-generator.conf - default='placement-policy.yaml', - help='The file that defines placement policies. This can be an ' - 'absolute path or relative to the configuration file.'), - cfg.StrOpt( - 'incomplete_consumer_project_id', - default=DEFAULT_CONSUMER_MISSING_ID, - help=""" -Early API microversions (<1.8) allowed creating allocations and not specifying -a project or user identifier for the consumer. In cleaning up the data -modeling, we no longer allow missing project and user information. If an older -client makes an allocation, we'll use this in place of the information it -doesn't provide. -"""), - cfg.StrOpt( - 'incomplete_consumer_user_id', - default=DEFAULT_CONSUMER_MISSING_ID, - help=""" -Early API microversions (<1.8) allowed creating allocations and not specifying -a project or user identifier for the consumer. In cleaning up the data -modeling, we no longer allow missing project and user information. If an older -client makes an allocation, we'll use this in place of the information it -doesn't provide. -"""), -] - - -# Duplicate log_options from oslo_service so that we don't have to import -# that package into placement. -# NOTE(cdent): Doing so ends up requiring eventlet and other unnecessary -# packages for just this one setting. -service_opts = [ - cfg.BoolOpt('log_options', - default=True, - help='Enables or disables logging values of all registered ' - 'options when starting a service (at DEBUG level).'), -] - def register_opts(conf): conf.register_group(placement_group) - conf.register_opts(placement_opts, group=placement_group) - conf.register_opts(service_opts) confutils.register_ksa_opts(conf, placement_group, DEFAULT_SERVICE_TYPE) def list_opts(): return { placement_group.name: ( - placement_opts + ks_loading.get_session_conf_options() + ks_loading.get_auth_common_conf_options() + ks_loading.get_auth_plugin_conf_options('password') + diff --git a/nova/config.py b/nova/config.py index de2d10c8264f..b5064b4783f7 100644 --- a/nova/config.py +++ b/nova/config.py @@ -18,7 +18,6 @@ from oslo_log import log from oslo_utils import importutils -from nova.api.openstack.placement import db_api as placement_db from nova.common import config import nova.conf from nova.db.sqlalchemy import api as sqlalchemy_api @@ -62,4 +61,3 @@ def parse_args(argv, default_config_files=None, configure_db=True, if configure_db: sqlalchemy_api.configure(CONF) - placement_db.configure(CONF) diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py index 79d8423a07a1..a8a544999f0a 100644 --- a/nova/db/sqlalchemy/migration.py +++ b/nova/db/sqlalchemy/migration.py @@ -24,7 +24,6 @@ from oslo_log import log as logging import sqlalchemy from sqlalchemy.sql import null -from nova.api.openstack.placement import db_api as placement_db from nova.db.sqlalchemy import api as db_session from nova import exception from nova.i18n import _ @@ -32,7 +31,6 @@ from nova.i18n import _ INIT_VERSION = {} INIT_VERSION['main'] = 215 INIT_VERSION['api'] = 0 -INIT_VERSION['placement'] = 0 _REPOSITORY = {} LOG = logging.getLogger(__name__) @@ -43,8 +41,6 @@ def get_engine(database='main', context=None): return db_session.get_engine(context=context) if database == 'api': return db_session.get_api_engine() - if database == 'placement': - return placement_db.get_placement_engine() def db_sync(version=None, database='main', context=None): @@ -173,10 +169,7 @@ def _find_migrate_repo(database='main'): """Get the path for the migrate repository.""" global _REPOSITORY rel_path = 'migrate_repo' - if database == 'api' or database == 'placement': - # NOTE(cdent): For the time being the placement database (if - # it is being used) is a replica (in structure) of the api - # database. + if database == 'api': rel_path = os.path.join('api_migrations', 'migrate_repo') path = os.path.join(os.path.abspath(os.path.dirname(__file__)), rel_path) diff --git a/nova/hacking/checks.py b/nova/hacking/checks.py index b567d5c44d2c..0f3cf06572f2 100644 --- a/nova/hacking/checks.py +++ b/nova/hacking/checks.py @@ -626,15 +626,12 @@ def check_config_option_in_central_place(logical_line, filename): def check_policy_registration_in_central_place(logical_line, filename): msg = ('N350: Policy registration should be in the central location(s) ' - '"/nova/policies/*" or "nova/api/openstack/placement/policies/*".') + '"/nova/policies/*"') # This is where registration should happen - if ("nova/policies/" in filename or - "nova/api/openstack/placement/policies/" in filename): + if "nova/policies/" in filename: return # A couple of policy tests register rules - if ("nova/tests/unit/test_policy.py" in filename or - "nova/tests/unit/api/openstack/placement/test_policy.py" in - filename): + if "nova/tests/unit/test_policy.py" in filename: return if rule_default_re.match(logical_line): diff --git a/nova/rc_fields.py b/nova/rc_fields.py deleted file mode 100644 index 5c525e9214ca..000000000000 --- a/nova/rc_fields.py +++ /dev/null @@ -1,70 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Standard Resource Class Fields.""" - -# NOTE(cdent): This file is only used by the placement code within -# nova. Other uses of resource classes in nova make use of the -# os-resource-classes library. The placement code within nova -# continues to use this so that that code can remain unchanged. - -import re - -from oslo_versionedobjects import fields - - -class ResourceClass(fields.StringField): - """Classes of resources provided to consumers.""" - - CUSTOM_NAMESPACE = 'CUSTOM_' - """All non-standard resource classes must begin with this string.""" - - VCPU = 'VCPU' - MEMORY_MB = 'MEMORY_MB' - DISK_GB = 'DISK_GB' - PCI_DEVICE = 'PCI_DEVICE' - SRIOV_NET_VF = 'SRIOV_NET_VF' - NUMA_SOCKET = 'NUMA_SOCKET' - NUMA_CORE = 'NUMA_CORE' - NUMA_THREAD = 'NUMA_THREAD' - NUMA_MEMORY_MB = 'NUMA_MEMORY_MB' - IPV4_ADDRESS = 'IPV4_ADDRESS' - VGPU = 'VGPU' - VGPU_DISPLAY_HEAD = 'VGPU_DISPLAY_HEAD' - # Standard resource class for network bandwidth egress measured in - # kilobits per second. - NET_BW_EGR_KILOBIT_PER_SEC = 'NET_BW_EGR_KILOBIT_PER_SEC' - # Standard resource class for network bandwidth ingress measured in - # kilobits per second. - NET_BW_IGR_KILOBIT_PER_SEC = 'NET_BW_IGR_KILOBIT_PER_SEC' - - # The ordering here is relevant. If you must add a value, only - # append. - STANDARD = (VCPU, MEMORY_MB, DISK_GB, PCI_DEVICE, SRIOV_NET_VF, - NUMA_SOCKET, NUMA_CORE, NUMA_THREAD, NUMA_MEMORY_MB, - IPV4_ADDRESS, VGPU, VGPU_DISPLAY_HEAD, - NET_BW_EGR_KILOBIT_PER_SEC, NET_BW_IGR_KILOBIT_PER_SEC) - - @classmethod - def normalize_name(cls, rc_name): - if rc_name is None: - return None - # Replace non-alphanumeric characters with underscores - norm_name = re.sub('[^0-9A-Za-z]+', '_', rc_name) - # Bug #1762789: Do .upper after replacing non alphanumerics. - norm_name = norm_name.upper() - norm_name = cls.CUSTOM_NAMESPACE + norm_name - return norm_name - - -class ResourceClassField(fields.AutoTypedField): - AUTO_TYPE = ResourceClass() diff --git a/nova/tests/functional/test_nova_manage.py b/nova/tests/functional/test_nova_manage.py index f5c2038af701..444db78a4144 100644 --- a/nova/tests/functional/test_nova_manage.py +++ b/nova/tests/functional/test_nova_manage.py @@ -24,6 +24,7 @@ from nova import test from nova.tests.functional import integrated_helpers CONF = config.CONF +INCOMPLETE_CONSUMER_ID = '00000000-0000-0000-0000-000000000000' class NovaManageDBIronicTest(test.TestCase): @@ -626,10 +627,8 @@ class TestNovaManagePlacementHealAllocations( # the project_id and user_id are based on the sentinel values. allocations = self.placement_api.get( '/allocations/%s' % server['id'], version='1.12').body - self.assertEqual(CONF.placement.incomplete_consumer_project_id, - allocations['project_id']) - self.assertEqual(CONF.placement.incomplete_consumer_user_id, - allocations['user_id']) + self.assertEqual(INCOMPLETE_CONSUMER_ID, allocations['project_id']) + self.assertEqual(INCOMPLETE_CONSUMER_ID, allocations['user_id']) allocations = allocations['allocations'] self.assertIn(rp_uuid, allocations) self.assertFlavorMatchesAllocation(self.flavor, server['id'], rp_uuid) diff --git a/nova/tests/unit/policy_fixture.py b/nova/tests/unit/policy_fixture.py index 651f096bcba4..a076afa93d02 100644 --- a/nova/tests/unit/policy_fixture.py +++ b/nova/tests/unit/policy_fixture.py @@ -18,7 +18,6 @@ import fixtures from oslo_policy import policy as oslo_policy from oslo_serialization import jsonutils -from nova.api.openstack.placement import policy as placement_policy import nova.conf from nova.conf import paths from nova import policies @@ -127,32 +126,3 @@ class RoleBasedPolicyFixture(RealPolicyFixture): self.policy_file = os.path.join(self.policy_dir.path, 'policy.json') with open(self.policy_file, 'w') as f: jsonutils.dump(policy, f) - - -class PlacementPolicyFixture(fixtures.Fixture): - """Load the default placement policy for tests. - - This fixture requires nova.tests.unit.conf_fixture.ConfFixture. - """ - def setUp(self): - super(PlacementPolicyFixture, self).setUp() - policy_file = paths.state_path_def('etc/nova/placement-policy.yaml') - CONF.set_override('policy_file', policy_file, group='placement') - placement_policy.reset() - placement_policy.init() - self.addCleanup(placement_policy.reset) - - @staticmethod - def set_rules(rules, overwrite=True): - """Set placement policy rules. - - .. note:: The rules must first be registered via the - Enforcer.register_defaults method. - - :param rules: dict of action=rule mappings to set - :param overwrite: Whether to overwrite current rules or update them - with the new rules. - """ - enforcer = placement_policy.get_enforcer() - enforcer.set_rules(oslo_policy.Rules.from_dict(rules), - overwrite=overwrite) diff --git a/nova/tests/unit/test_conf.py b/nova/tests/unit/test_conf.py index 95f3ec41e966..21e5f730b44a 100644 --- a/nova/tests/unit/test_conf.py +++ b/nova/tests/unit/test_conf.py @@ -91,9 +91,6 @@ class TestParseArgs(test.NoDBTestCase): m = mock.patch('nova.db.sqlalchemy.api.configure') self.nova_db_config_mock = m.start() self.addCleanup(self.nova_db_config_mock.stop) - m = mock.patch('nova.api.openstack.placement.db_api.configure') - self.placement_db_config_mock = m.start() - self.addCleanup(self.placement_db_config_mock.stop) @mock.patch.object(config.log, 'register_options') def test_parse_args_glance_debug_false(self, register_options): @@ -101,7 +98,6 @@ class TestParseArgs(test.NoDBTestCase): config.parse_args([], configure_db=False, init_rpc=False) self.assertIn('glanceclient=WARN', config.CONF.default_log_levels) self.nova_db_config_mock.assert_not_called() - self.placement_db_config_mock.assert_not_called() @mock.patch.object(config.log, 'register_options') def test_parse_args_glance_debug_true(self, register_options): @@ -109,4 +105,3 @@ class TestParseArgs(test.NoDBTestCase): config.parse_args([], configure_db=True, init_rpc=False) self.assertIn('glanceclient=DEBUG', config.CONF.default_log_levels) self.nova_db_config_mock.assert_called_once_with(config.CONF) - self.placement_db_config_mock.assert_called_once_with(config.CONF) diff --git a/nova/tests/unit/test_nova_manage.py b/nova/tests/unit/test_nova_manage.py index a00e4b45bd44..393616bb22b6 100644 --- a/nova/tests/unit/test_nova_manage.py +++ b/nova/tests/unit/test_nova_manage.py @@ -2562,7 +2562,7 @@ class TestNovaManagePlacement(test.NoDBTestCase): new_callable=mock.NonCallableMock) # assert not called @mock.patch('nova.scheduler.client.report.SchedulerReportClient.put', return_value=fake_requests.FakeResponse(204)) - def test_heal_allocations_sentinel_consumer( + def test_heal_allocations( self, mock_put, mock_get_compute_node, mock_get_allocs, mock_get_instances, mock_get_all_cells): """Tests the scenario that there are allocations created using @@ -2584,8 +2584,8 @@ class TestNovaManagePlacement(test.NoDBTestCase): } } }, - "project_id": CONF.placement.incomplete_consumer_project_id, - "user_id": CONF.placement.incomplete_consumer_user_id + "project_id": uuidsentinel.project_id, + "user_id": uuidsentinel.user_id } self.assertEqual(0, self.cli.heal_allocations(verbose=True)) self.assertIn('Processed 1 instances.', self.output.getvalue()) @@ -2614,7 +2614,7 @@ class TestNovaManagePlacement(test.NoDBTestCase): return_value=fake_requests.FakeResponse( 409, content='Inventory and/or allocations changed while ' 'attempting to allocate')) - def test_heal_allocations_sentinel_consumer_put_fails( + def test_heal_allocations_put_fails( self, mock_put, mock_get_allocs, mock_get_instances, mock_get_all_cells): """Tests the scenario that there are allocations created using @@ -2634,8 +2634,8 @@ class TestNovaManagePlacement(test.NoDBTestCase): } } }, - "project_id": CONF.placement.incomplete_consumer_project_id, - "user_id": CONF.placement.incomplete_consumer_user_id + "project_id": uuidsentinel.project_id, + "user_id": uuidsentinel.user_id } self.assertEqual(3, self.cli.heal_allocations(verbose=True)) self.assertIn( diff --git a/releasenotes/notes/placement-deleted-a79ad405f428a5f8.yaml b/releasenotes/notes/placement-deleted-a79ad405f428a5f8.yaml new file mode 100644 index 000000000000..416c8f2fa023 --- /dev/null +++ b/releasenotes/notes/placement-deleted-a79ad405f428a5f8.yaml @@ -0,0 +1,13 @@ +--- +other: + - | + The code for the `placement service + `_ was moved to its own + `repository `_ in + Stein. The placement code in nova has been deleted. +upgrade: + - | + If you upgraded your OpenStack deployment to Stein without switching to use + the now independent placement service, you must do so before upgrading to + Train. `Instructions `_ + for one way to do this are available. diff --git a/setup.cfg b/setup.cfg index b9481bc445e5..985191dc82bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,6 @@ oslo.config.opts.defaults = oslo.policy.enforcer = nova = nova.policy:get_enforcer - placement = nova.api.openstack.placement.policy:get_enforcer oslo.policy.policies = # The sample policies will be ordered by entry point and then by list @@ -48,7 +47,6 @@ oslo.policy.policies = # list_rules method into a separate entry point rather than using the # aggregate method. nova = nova.policies:list_rules - placement = nova.api.openstack.placement.policies:list_rules nova.compute.monitors.cpu = virt_driver = nova.compute.monitors.cpu.virt_driver:Monitor @@ -74,7 +72,6 @@ console_scripts = nova-status = nova.cmd.status:main nova-xvpvncproxy = nova.cmd.xvpvncproxy:main wsgi_scripts = - nova-placement-api = nova.api.openstack.placement.wsgi:init_application nova-api-wsgi = nova.api.openstack.compute.wsgi:init_application nova-metadata-wsgi = nova.api.metadata.wsgi:init_application diff --git a/tox.ini b/tox.ini index 2e1af697862f..65effb9b175a 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,7 @@ commands = # special way. See the following for more details. # http://stestr.readthedocs.io/en/latest/MANUAL.html#grouping-tests # https://gabbi.readthedocs.io/en/latest/#purpose - stestr --test-path=./nova/tests/functional --group_regex=nova\.tests\.functional\.api\.openstack\.placement\.test_placement_api(?:\.|_)([^_]+) run {posargs} + stestr --test-path=./nova/tests/functional run {posargs} stestr slowest # TODO(gcb) Merge this into [testenv:functional] when functional tests are gating @@ -132,11 +132,6 @@ envdir = {toxworkdir}/shared commands = oslopolicy-sample-generator --config-file=etc/nova/nova-policy-generator.conf -[testenv:genplacementpolicy] -envdir = {toxworkdir}/shared -commands = - oslopolicy-sample-generator --config-file=etc/nova/placement-policy-generator.conf - [testenv:cover] # TODO(stephenfin): Remove the PYTHON hack below in favour of a [coverage] # section once we rely on coverage 4.3+