diff --git a/doc/source/devref/effective_neutron.rst b/doc/source/devref/effective_neutron.rst index 57429683467..d8c9bf97410 100644 --- a/doc/source/devref/effective_neutron.rst +++ b/doc/source/devref/effective_neutron.rst @@ -159,7 +159,7 @@ Document common pitfalls as well as good practices done during database developm section because internal transaction from ``_do_other_thing_with_created_object`` has been rolled back. To avoid this nested transactions should be used. For such cases help function ``safe_creation`` has been created in - ``neutron/db/common_db_mixin.py``. + ``neutron/db/_utils.py``. So, the example above should be replaced with: .. code:: python diff --git a/neutron/db/_utils.py b/neutron/db/_utils.py new file mode 100644 index 00000000000..5a5ee0f0e58 --- /dev/null +++ b/neutron/db/_utils.py @@ -0,0 +1,123 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +NOTE: This module shall not be used by external projects. It will be moved + to neutron-lib in due course, and then it can be used from there. +""" + +import contextlib + +from oslo_log import log as logging +from oslo_utils import excutils +import six +from sqlalchemy.ext import associationproxy + +from neutron._i18n import _LE +from neutron.api.v2 import attributes + +LOG = logging.getLogger(__name__) + + +@contextlib.contextmanager +def _noop_context_manager(): + yield + + +def safe_creation(context, create_fn, delete_fn, create_bindings, + transaction=True): + '''This function wraps logic of object creation in safe atomic way. + + In case of exception, object is deleted. + + More information when this method could be used can be found in + developer guide - Effective Neutron: Database interaction section. + http://docs.openstack.org/developer/neutron/devref/effective_neutron.html + + :param context: context + + :param create_fn: function without arguments that is called to create + object and returns this object. + + :param delete_fn: function that is called to delete an object. It is + called with object's id field as an argument. + + :param create_bindings: function that is called to create bindings for + an object. It is called with object's id field as an argument. + + :param transaction: if true the whole operation will be wrapped in a + transaction. if false, no transaction will be used. + ''' + cm = (context.session.begin(subtransactions=True) + if transaction else _noop_context_manager()) + with cm: + obj = create_fn() + try: + value = create_bindings(obj['id']) + except Exception: + with excutils.save_and_reraise_exception(): + try: + delete_fn(obj['id']) + except Exception as e: + LOG.error(_LE("Cannot clean up created object %(obj)s. " + "Exception: %(exc)s"), {'obj': obj['id'], + 'exc': e}) + return obj, value + + +def model_query_scope_is_project(context, model): + # Unless a context has 'admin' or 'advanced-service' rights the + # query will be scoped to a single project_id + return ((not context.is_admin and hasattr(model, 'project_id')) and + (not context.is_advsvc and hasattr(model, 'project_id'))) + + +def model_query(context, model): + query = context.session.query(model) + # define basic filter condition for model query + query_filter = None + if model_query_scope_is_project(context, model): + query_filter = (model.tenant_id == context.tenant_id) + + if query_filter is not None: + query = query.filter(query_filter) + return query + + +# NOTE: This used to be CommonDbMixin._fields() +def resource_fields(resource, fields): + """Return only the resource items that are in fields. + + :param resource: a resource dictionary + :type resource: dict + :param fields: a list of fields to select from the resource + :type fields: list + + """ + if fields: + resource = {key: item for key, item in resource.items() + if key in fields} + return attributes.populate_project_info(resource) + + +# NOTE: This used to be CommonDbMixin._filter_non_model_columns +def filter_non_model_columns(data, model): + """Return the attributes from data which are model columns. + + Return a new dict with items from data that whose keys are columns in + the model or are association proxies of the model. + """ + columns = [c.name for c in model.__table__.columns] + return dict((k, v) for (k, v) in + six.iteritems(data) if k in columns or + isinstance(getattr(model, k, None), + associationproxy.AssociationProxy)) diff --git a/neutron/db/address_scope_db.py b/neutron/db/address_scope_db.py index 4038b96202d..285afe7244d 100644 --- a/neutron/db/address_scope_db.py +++ b/neutron/db/address_scope_db.py @@ -19,6 +19,7 @@ from sqlalchemy.orm import exc from neutron._i18n import _ from neutron.api.v2 import attributes as attr from neutron.common import _deprecate +from neutron.db import _utils as db_utils from neutron.db import db_base_plugin_v2 from neutron.db.models import address_scope as address_scope_model from neutron.extensions import address_scope as ext_address_scope @@ -33,13 +34,14 @@ class AddressScopeDbMixin(ext_address_scope.AddressScopePluginBase): __native_bulk_support = True - def _make_address_scope_dict(self, address_scope, fields=None): + @staticmethod + def _make_address_scope_dict(address_scope, fields=None): res = {'id': address_scope['id'], 'name': address_scope['name'], 'tenant_id': address_scope['tenant_id'], 'shared': address_scope['shared'], 'ip_version': address_scope['ip_version']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _get_address_scope(self, context, id): try: diff --git a/neutron/db/agents_db.py b/neutron/db/agents_db.py index 1ecb934e16b..3492b6fbeac 100644 --- a/neutron/db/agents_db.py +++ b/neutron/db/agents_db.py @@ -38,6 +38,7 @@ from neutron.callbacks import resources from neutron.common import _deprecate from neutron.common import constants as n_const from neutron import context +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db.models import agent as agent_model from neutron.extensions import agent as ext_agent @@ -226,7 +227,7 @@ class AgentDbMixin(ext_agent.AgentPluginBase, AgentAvailabilityZoneMixin): res['resource_versions'] = self._get_dict(agent, 'resource_versions', ignore_missing=True) res['availability_zone'] = agent['availability_zone'] - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) @db_api.retry_if_session_inactive() def delete_agent(self, context, id): diff --git a/neutron/db/allowedaddresspairs_db.py b/neutron/db/allowedaddresspairs_db.py index aceae0a5b06..1d9464ecbfc 100644 --- a/neutron/db/allowedaddresspairs_db.py +++ b/neutron/db/allowedaddresspairs_db.py @@ -16,6 +16,7 @@ from neutron_lib.api import validators from neutron.api.v2 import attributes as attr +from neutron.db import _utils as db_utils from neutron.db import db_base_plugin_v2 from neutron.common import utils @@ -86,11 +87,12 @@ class AllowedAddressPairsMixin(object): for pair in pairs: pair.delete() - def _make_allowed_address_pairs_dict(self, allowed_address_pairs, + @staticmethod + def _make_allowed_address_pairs_dict(allowed_address_pairs, fields=None): res = {'mac_address': allowed_address_pairs['mac_address'], 'ip_address': allowed_address_pairs['ip_address']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _has_address_pairs(self, port): return (validators.is_attr_set(port['port'][addr_pair.ADDRESS_PAIRS]) diff --git a/neutron/db/common_db_mixin.py b/neutron/db/common_db_mixin.py index 5efdc09fd2a..49f400a647c 100644 --- a/neutron/db/common_db_mixin.py +++ b/neutron/db/common_db_mixin.py @@ -13,89 +13,28 @@ # License for the specific language governing permissions and limitations # under the License. -import contextlib import weakref from neutron_lib.db import utils as db_utils from oslo_db.sqlalchemy import utils as sa_utils from oslo_log import log as logging -from oslo_utils import excutils import six from sqlalchemy import and_ from sqlalchemy.ext import associationproxy from sqlalchemy import or_ from sqlalchemy import sql -from neutron._i18n import _LE from neutron.api.v2 import attributes - +from neutron.db import _utils as ndb_utils LOG = logging.getLogger(__name__) -@contextlib.contextmanager -def _noop_context_manager(): - yield - - -def safe_creation(context, create_fn, delete_fn, create_bindings, - transaction=True): - '''This function wraps logic of object creation in safe atomic way. - - In case of exception, object is deleted. - - More information when this method could be used can be found in - developer guide - Effective Neutron: Database interaction section. - http://docs.openstack.org/developer/neutron/devref/effective_neutron.html - - :param context: context - - :param create_fn: function without arguments that is called to create - object and returns this object. - - :param delete_fn: function that is called to delete an object. It is - called with object's id field as an argument. - - :param create_bindings: function that is called to create bindings for - an object. It is called with object's id field as an argument. - - :param transaction: if true the whole operation will be wrapped in a - transaction. if false, no transaction will be used. - ''' - cm = (context.session.begin(subtransactions=True) - if transaction else _noop_context_manager()) - with cm: - obj = create_fn() - try: - value = create_bindings(obj['id']) - except Exception: - with excutils.save_and_reraise_exception(): - try: - delete_fn(obj['id']) - except Exception as e: - LOG.error(_LE("Cannot clean up created object %(obj)s. " - "Exception: %(exc)s"), {'obj': obj['id'], - 'exc': e}) - return obj, value - - -def model_query_scope(context, model): - # Unless a context has 'admin' or 'advanced-service' rights the - # query will be scoped to a single tenant_id - return ((not context.is_admin and hasattr(model, 'tenant_id')) and - (not context.is_advsvc and hasattr(model, 'tenant_id'))) - - -def model_query(context, model): - query = context.session.query(model) - # define basic filter condition for model query - query_filter = None - if model_query_scope(context, model): - query_filter = (model.tenant_id == context.tenant_id) - - if query_filter is not None: - query = query.filter(query_filter) - return query +# TODO(HenryG): Remove these when available in neutron-lib +safe_creation = ndb_utils.safe_creation +model_query_scope = ndb_utils.model_query_scope_is_project +model_query = ndb_utils.model_query +resource_fields = ndb_utils.resource_fields class CommonDbMixin(object): @@ -147,14 +86,15 @@ class CommonDbMixin(object): """ return weakref.proxy(self) + # TODO(HenryG): Remove this when available in neutron-lib def model_query_scope(self, context, model): - return model_query_scope(context, model) + return ndb_utils.model_query_scope_is_project(context, model) def _model_query(self, context, model): query = context.session.query(model) # define basic filter condition for model query query_filter = None - if self.model_query_scope(context, model): + if ndb_utils.model_query_scope_is_project(context, model): if hasattr(model, 'rbac_entries'): query = query.outerjoin(model.rbac_entries) rbac_model = model.rbac_entries.property.mapper.class_ @@ -189,11 +129,9 @@ class CommonDbMixin(object): query = query.filter(query_filter) return query + # TODO(HenryG): Remove this when available in neutron-lib def _fields(self, resource, fields): - if fields: - resource = {key: item for key, item in resource.items() - if key in fields} - return attributes.populate_project_info(resource) + return ndb_utils.resource_fields(resource, fields) def _get_by_id(self, context, model, id): query = self._model_query(context, model) @@ -315,12 +253,6 @@ class CommonDbMixin(object): return getattr(self, '_get_%s' % resource)(context, marker) return None + # TODO(HenryG): Remove this when available in neutron-lib def _filter_non_model_columns(self, data, model): - """Remove all the attributes from data which are not columns or - association proxies of the model passed as second parameter - """ - columns = [c.name for c in model.__table__.columns] - return dict((k, v) for (k, v) in - six.iteritems(data) if k in columns or - isinstance(getattr(model, k, None), - associationproxy.AssociationProxy)) + return ndb_utils.filter_non_model_columns(data, model) diff --git a/neutron/db/db_base_plugin_common.py b/neutron/db/db_base_plugin_common.py index 8616b61caca..a20605b181f 100644 --- a/neutron/db/db_base_plugin_common.py +++ b/neutron/db/db_base_plugin_common.py @@ -26,6 +26,7 @@ from neutron.api.v2 import attributes from neutron.common import constants as n_const from neutron.common import exceptions from neutron.common import utils +from neutron.db import _utils as db_utils from neutron.db import common_db_mixin from neutron.db import models_v2 from neutron.objects import subnet as subnet_obj @@ -153,7 +154,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): res['shared'] = self._is_network_shared(context, subnet.networks) # Call auxiliary extend functions, if any self._apply_dict_extend_functions(attributes.SUBNETS, res, subnet) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _make_subnetpool_dict(self, subnetpool, fields=None): default_prefixlen = str(subnetpool['default_prefixlen']) @@ -173,7 +174,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): 'address_scope_id': subnetpool['address_scope_id']} self._apply_dict_extend_functions(attributes.SUBNETPOOLS, res, subnetpool) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _make_port_dict(self, port, fields=None, process_extensions=True): @@ -193,7 +194,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): if process_extensions: self._apply_dict_extend_functions( attributes.PORTS, res, port) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _get_network(self, context, id): try: @@ -278,7 +279,7 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): if process_extensions: self._apply_dict_extend_functions( attributes.NETWORKS, res, network) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _is_network_shared(self, context, network): # The shared attribute for a network now reflects if the network @@ -315,20 +316,22 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): 'ip_address': ip["ip_address"]} for ip in ips] - def _port_filter_hook(self, context, original_model, conditions): + @staticmethod + def _port_filter_hook(context, original_model, conditions): # Apply the port filter only in non-admin and non-advsvc context - if self.model_query_scope(context, original_model): + if db_utils.model_query_scope_is_project(context, original_model): conditions |= ( (context.tenant_id == models_v2.Network.tenant_id) & (models_v2.Network.id == models_v2.Port.network_id)) return conditions - def _port_query_hook(self, context, original_model, query): + @staticmethod + def _port_query_hook(context, original_model, query): # we need to outerjoin to networks if the model query scope # is necessary so we can filter based on network id. without # this the conditions in the filter hook cause the networks # table to be added to the FROM statement so we get lots of # duplicated rows that break the COUNT operation - if self.model_query_scope(context, original_model): + if db_utils.model_query_scope_is_project(context, original_model): query = query.outerjoin(models_v2.Network) return query diff --git a/neutron/db/db_base_plugin_v2.py b/neutron/db/db_base_plugin_v2.py index a13bb83ff41..4b91d508789 100644 --- a/neutron/db/db_base_plugin_v2.py +++ b/neutron/db/db_base_plugin_v2.py @@ -42,6 +42,7 @@ from neutron.common import exceptions as n_exc from neutron.common import ipv6_utils from neutron.common import utils from neutron import context as ctx +from neutron.db import _utils as ndb_utils from neutron.db import api as db_api from neutron.db import db_base_plugin_common from neutron.db import ipam_pluggable_backend @@ -394,8 +395,8 @@ class NeutronDbPluginV2(db_base_plugin_common.DbBasePluginCommon, # The filter call removes attributes from the body received from # the API that are logically tied to network resources but are # stored in other database tables handled by extensions - network.update(self._filter_non_model_columns(n, - models_v2.Network)) + network.update( + ndb_utils.filter_non_model_columns(n, models_v2.Network)) return self._make_network_dict(network, context=context) @db_api.retry_if_session_inactive() diff --git a/neutron/db/external_net_db.py b/neutron/db/external_net_db.py index 9e852a819a1..fe00f57c4ca 100644 --- a/neutron/db/external_net_db.py +++ b/neutron/db/external_net_db.py @@ -26,6 +26,7 @@ from neutron.callbacks import exceptions as c_exc from neutron.callbacks import registry from neutron.callbacks import resources from neutron.common import _deprecate +from neutron.db import _utils as db_utils from neutron.db import db_base_plugin_v2 from neutron.db.models import external_net as ext_net_models from neutron.db.models import l3 as l3_models @@ -51,12 +52,13 @@ class External_net_db_mixin(object): ext_net_models.ExternalNetwork.network_id)) return query - def _network_filter_hook(self, context, original_model, conditions): + @staticmethod + def _network_filter_hook(context, original_model, conditions): if conditions is not None and not hasattr(conditions, '__iter__'): conditions = (conditions, ) # Apply the external network filter only in non-admin and non-advsvc # context - if self.model_query_scope(context, original_model): + if db_utils.model_query_scope_is_project(context, original_model): # the table will already be joined to the rbac entries for the # shared check so we don't need to worry about ensuring that rbac_model = original_model.rbac_entries.property.mapper.class_ diff --git a/neutron/db/flavors_db.py b/neutron/db/flavors_db.py index 23b30040978..f4a1416c615 100644 --- a/neutron/db/flavors_db.py +++ b/neutron/db/flavors_db.py @@ -18,6 +18,7 @@ from oslo_utils import uuidutils from sqlalchemy.orm import exc as sa_exc from neutron.common import _deprecate +from neutron.db import _utils as db_utils from neutron.db import common_db_mixin from neutron.db.models import flavor as flavor_models from neutron.db import servicetype_db as sdb @@ -49,7 +50,8 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): except sa_exc.NoResultFound: raise ext_flavors.ServiceProfileNotFound(sp_id=sp_id) - def _make_flavor_dict(self, flavor_db, fields=None): + @staticmethod + def _make_flavor_dict(flavor_db, fields=None): res = {'id': flavor_db['id'], 'name': flavor_db['name'], 'description': flavor_db['description'], @@ -59,9 +61,10 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): if flavor_db.service_profiles: res['service_profiles'] = [sp['service_profile_id'] for sp in flavor_db.service_profiles] - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) - def _make_service_profile_dict(self, sp_db, fields=None): + @staticmethod + def _make_service_profile_dict(sp_db, fields=None): res = {'id': sp_db['id'], 'description': sp_db['description'], 'driver': sp_db['driver'], @@ -70,7 +73,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): if sp_db.flavors: res['flavors'] = [fl['flavor_id'] for fl in sp_db.flavors] - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _ensure_flavor_not_in_use(self, context, flavor_id): """Checks that flavor is not associated with service instance.""" @@ -177,7 +180,8 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): sp_id=service_profile_id, fl_id=flavor_id) context.session.delete(binding) - def get_flavor_service_profile(self, context, + @staticmethod + def get_flavor_service_profile(context, service_profile_id, flavor_id, fields=None): with context.session.begin(subtransactions=True): binding = ( @@ -191,7 +195,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): sp_id=service_profile_id, fl_id=flavor_id) res = {'service_profile_id': service_profile_id, 'flavor_id': flavor_id} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def create_service_profile(self, context, service_profile): sp = service_profile['service_profile'] @@ -283,7 +287,7 @@ class FlavorsDbMixin(common_db_mixin.CommonDbMixin): res = {'driver': sp_db.driver, 'provider': providers[0].get('name')} - return [self._fields(res, fields)] + return [db_utils.resource_fields(res, fields)] _deprecate._MovedGlobals() diff --git a/neutron/db/ipam_backend_mixin.py b/neutron/db/ipam_backend_mixin.py index 7b5b7ff3f06..f1236715f3f 100644 --- a/neutron/db/ipam_backend_mixin.py +++ b/neutron/db/ipam_backend_mixin.py @@ -32,6 +32,7 @@ from neutron.common import constants from neutron.common import exceptions as n_exc from neutron.common import ipv6_utils from neutron.common import utils as common_utils +from neutron.db import _utils as db_utils from neutron.db import db_base_plugin_common from neutron.db.models import segment as segment_model from neutron.db.models import subnet_service_type as sst_model @@ -107,8 +108,8 @@ class IpamBackendMixin(db_base_plugin_common.DbBasePluginCommon): if (new_mac and new_mac != db_port.mac_address and self._is_mac_in_use(context, network_id, new_mac)): raise exc.MacAddressInUse(net_id=network_id, mac=new_mac) - db_port.update(self._filter_non_model_columns(new_port, - models_v2.Port)) + db_port.update(db_utils.filter_non_model_columns(new_port, + models_v2.Port)) def _update_subnet_host_routes(self, context, id, s): diff --git a/neutron/db/l3_db.py b/neutron/db/l3_db.py index 635fe7f37a4..da7db332c13 100644 --- a/neutron/db/l3_db.py +++ b/neutron/db/l3_db.py @@ -38,6 +38,7 @@ from neutron.common import constants as n_const from neutron.common import ipv6_utils from neutron.common import rpc as n_rpc from neutron.common import utils +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db.models import l3 as l3_models @@ -135,7 +136,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase, # plugins. if process_extensions: self._apply_dict_extend_functions(l3.ROUTERS, res, router) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def filter_allocating_and_missing_routers(self, context, routers): """Filter out routers that shouldn't go to the agent. @@ -974,7 +975,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase, # plugins. if process_extensions: self._apply_dict_extend_functions(l3.FLOATINGIPS, res, floatingip) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _get_router_for_floatingip(self, context, internal_port, internal_subnet_id, diff --git a/neutron/db/l3_hamode_db.py b/neutron/db/l3_hamode_db.py index 96328bbc0ee..79b5b3f931e 100644 --- a/neutron/db/l3_hamode_db.py +++ b/neutron/db/l3_hamode_db.py @@ -34,9 +34,9 @@ from neutron.api.v2 import attributes from neutron.common import _deprecate from neutron.common import constants as n_const from neutron.common import utils as n_utils +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db.availability_zone import router as router_az_db -from neutron.db import common_db_mixin from neutron.db import l3_dvr_db from neutron.db.l3_dvr_db import is_distributed_router from neutron.db.models import agent as agent_model @@ -246,7 +246,7 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin, deletion = functools.partial(self._core_plugin.delete_network, admin_ctx) - network, ha_network = common_db_mixin.safe_creation( + network, ha_network = db_utils.safe_creation( context, creation, deletion, content, transaction=False) try: self._create_ha_subnet(admin_ctx, network['id'], tenant_id) @@ -327,9 +327,9 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin, router_id) deletion = functools.partial(self._core_plugin.delete_port, context, l3_port_check=False) - port, bindings = common_db_mixin.safe_creation(context, creation, - deletion, content, - transaction=False) + port, bindings = db_utils.safe_creation(context, creation, + deletion, content, + transaction=False) return bindings def _create_ha_interfaces(self, context, router, ha_network): diff --git a/neutron/db/metering/metering_db.py b/neutron/db/metering/metering_db.py index f8e8828dada..fe6a1e9f6c1 100644 --- a/neutron/db/metering/metering_db.py +++ b/neutron/db/metering/metering_db.py @@ -19,6 +19,7 @@ from sqlalchemy import orm from neutron.api.rpc.agentnotifiers import metering_rpc_agent_api from neutron.common import _deprecate from neutron.common import constants +from neutron.db import _utils as db_utils from neutron.db import common_db_mixin as base_db from neutron.db.models import l3 as l3_models from neutron.db.models import metering as metering_models @@ -35,13 +36,14 @@ class MeteringDbMixin(metering.MeteringPluginBase, def __init__(self): self.meter_rpc = metering_rpc_agent_api.MeteringAgentNotifyAPI() - def _make_metering_label_dict(self, metering_label, fields=None): + @staticmethod + def _make_metering_label_dict(metering_label, fields=None): res = {'id': metering_label['id'], 'name': metering_label['name'], 'description': metering_label['description'], 'shared': metering_label['shared'], 'tenant_id': metering_label['tenant_id']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def create_metering_label(self, context, metering_label): m = metering_label['metering_label'] @@ -91,13 +93,14 @@ class MeteringDbMixin(metering.MeteringPluginBase, marker_obj=marker_obj, page_reverse=page_reverse) - def _make_metering_label_rule_dict(self, metering_label_rule, fields=None): + @staticmethod + def _make_metering_label_rule_dict(metering_label_rule, fields=None): res = {'id': metering_label_rule['id'], 'metering_label_id': metering_label_rule['metering_label_id'], 'direction': metering_label_rule['direction'], 'remote_ip_prefix': metering_label_rule['remote_ip_prefix'], 'excluded': metering_label_rule['excluded']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def get_metering_label_rules(self, context, filters=None, fields=None, sorts=None, limit=None, marker=None, diff --git a/neutron/db/portsecurity_db_common.py b/neutron/db/portsecurity_db_common.py index 5bd8dad0d75..9e319f35893 100644 --- a/neutron/db/portsecurity_db_common.py +++ b/neutron/db/portsecurity_db_common.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron.db import _utils as db_utils from neutron.extensions import portsecurity as psec from neutron.objects import network from neutron.objects.port.extensions import port_security as p_ps @@ -91,7 +92,8 @@ class PortSecurityDbCommon(object): self._process_port_security_create( context, obj_cls, res_name, req, res) - def _make_port_security_dict(self, res, res_name, fields=None): + @staticmethod + def _make_port_security_dict(res, res_name, fields=None): res_ = {'%s_id' % res_name: res.id, psec.PORTSECURITY: res.port_security_enabled} - return self._fields(res_, fields) + return db_utils.resource_fields(res_, fields) diff --git a/neutron/db/qos/api.py b/neutron/db/qos/api.py index eb0aa6c9e8c..27ca266c2af 100644 --- a/neutron/db/qos/api.py +++ b/neutron/db/qos/api.py @@ -14,7 +14,7 @@ from oslo_db import exception as oslo_db_exception from sqlalchemy.orm import exc as orm_exc from neutron.common import exceptions as n_exc -from neutron.db import common_db_mixin as db +from neutron.db import _utils as db_utils from neutron.db.qos import models @@ -32,8 +32,8 @@ def create_policy_network_binding(context, policy_id, network_id): def delete_policy_network_binding(context, policy_id, network_id): try: with context.session.begin(subtransactions=True): - db_object = (db.model_query(context, - models.QosNetworkPolicyBinding) + db_object = (db_utils.model_query(context, + models.QosNetworkPolicyBinding) .filter_by(policy_id=policy_id, network_id=network_id).one()) context.session.delete(db_object) @@ -43,7 +43,7 @@ def delete_policy_network_binding(context, policy_id, network_id): def get_network_ids_by_network_policy_binding(context, policy_id): - query = (db.model_query(context, models.QosNetworkPolicyBinding) + query = (db_utils.model_query(context, models.QosNetworkPolicyBinding) .filter_by(policy_id=policy_id).all()) return [entry.network_id for entry in query] @@ -62,7 +62,8 @@ def create_policy_port_binding(context, policy_id, port_id): def delete_policy_port_binding(context, policy_id, port_id): try: with context.session.begin(subtransactions=True): - db_object = (db.model_query(context, models.QosPortPolicyBinding) + db_object = (db_utils.model_query(context, + models.QosPortPolicyBinding) .filter_by(policy_id=policy_id, port_id=port_id).one()) context.session.delete(db_object) @@ -72,6 +73,6 @@ def delete_policy_port_binding(context, policy_id, port_id): def get_port_ids_by_port_policy_binding(context, policy_id): - query = (db.model_query(context, models.QosPortPolicyBinding) + query = (db_utils.model_query(context, models.QosPortPolicyBinding) .filter_by(policy_id=policy_id).all()) return [entry.port_id for entry in query] diff --git a/neutron/db/quota/api.py b/neutron/db/quota/api.py index 49005dd228c..95034e8ffa1 100644 --- a/neutron/db/quota/api.py +++ b/neutron/db/quota/api.py @@ -19,8 +19,8 @@ import sqlalchemy as sa from sqlalchemy.orm import exc as orm_exc from sqlalchemy import sql +from neutron.db import _utils as db_utils from neutron.db import api as db_api -from neutron.db import common_db_mixin as common_db_api from neutron.db.quota import models as quota_models @@ -52,7 +52,7 @@ def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id, :returns: a QuotaUsageInfo instance """ - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource, tenant_id=tenant_id) if lock_for_update: @@ -69,7 +69,7 @@ def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id, @db_api.retry_if_session_inactive() def get_quota_usage_by_resource(context, resource): - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource) return [QuotaUsageInfo(item.resource, item.tenant_id, @@ -79,7 +79,7 @@ def get_quota_usage_by_resource(context, resource): @db_api.retry_if_session_inactive() def get_quota_usage_by_tenant_id(context, tenant_id): - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(tenant_id=tenant_id) return [QuotaUsageInfo(item.resource, item.tenant_id, @@ -102,7 +102,7 @@ def set_quota_usage(context, resource, tenant_id, or a delta (default to False) """ with db_api.autonested_transaction(context.session): - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource).filter_by( tenant_id=tenant_id) usage_data = query.first() @@ -135,7 +135,7 @@ def set_quota_usage_dirty(context, resource, tenant_id, dirty=True): :param dirty: the desired value for the dirty bit (defaults to True) :returns: 1 if the quota usage data were updated, 0 otherwise. """ - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id) return query.update({'dirty': dirty}) @@ -151,7 +151,7 @@ def set_resources_quota_usage_dirty(context, resources, tenant_id, dirty=True): :param dirty: the desired value for the dirty bit (defaults to True) :returns: the number of records for which the bit was actually set. """ - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(tenant_id=tenant_id) if resources: query = query.filter(quota_models.QuotaUsage.resource.in_(resources)) @@ -168,7 +168,7 @@ def set_all_quota_usage_dirty(context, resource, dirty=True): :returns: the number of tenants for which the dirty bit was actually updated """ - query = common_db_api.model_query(context, quota_models.QuotaUsage) + query = db_utils.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource) return query.update({'dirty': dirty}) diff --git a/neutron/db/quota/driver.py b/neutron/db/quota/driver.py index bf5b5798186..156570915c4 100644 --- a/neutron/db/quota/driver.py +++ b/neutron/db/quota/driver.py @@ -17,8 +17,8 @@ from neutron_lib import exceptions from oslo_log import log from neutron.common import exceptions as n_exc +from neutron.db import _utils as db_utils from neutron.db import api as db_api -from neutron.db import common_db_mixin as common_db from neutron.db.quota import api as quota_api from neutron.db.quota import models as quota_models @@ -66,7 +66,7 @@ class DbQuotaDriver(object): for key, resource in resources.items()) # update with tenant specific limits - q_qry = common_db.model_query(context, quota_models.Quota).filter_by( + q_qry = db_utils.model_query(context, quota_models.Quota).filter_by( tenant_id=tenant_id) for item in q_qry: tenant_quota[item['resource']] = item['limit'] diff --git a/neutron/db/rbac_db_mixin.py b/neutron/db/rbac_db_mixin.py index 67b3c3f1ba4..594ca57c5d9 100644 --- a/neutron/db/rbac_db_mixin.py +++ b/neutron/db/rbac_db_mixin.py @@ -20,6 +20,7 @@ from sqlalchemy.orm import exc from neutron.callbacks import events from neutron.callbacks import exceptions as c_exc from neutron.callbacks import registry +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import rbac_db_models as models @@ -56,11 +57,12 @@ class RbacPluginMixin(common_db_mixin.CommonDbMixin): raise ext_rbac.DuplicateRbacPolicy() return self._make_rbac_policy_dict(db_entry) - def _make_rbac_policy_dict(self, db_entry, fields=None): + @staticmethod + def _make_rbac_policy_dict(db_entry, fields=None): res = {f: db_entry[f] for f in ('id', 'tenant_id', 'target_tenant', 'action', 'object_id')} res['object_type'] = db_entry.object_type - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) @db_api.retry_if_session_inactive() def update_rbac_policy(self, context, id, rbac_policy): diff --git a/neutron/db/securitygroups_db.py b/neutron/db/securitygroups_db.py index 8a5aa3f5d48..8480dcb05d1 100644 --- a/neutron/db/securitygroups_db.py +++ b/neutron/db/securitygroups_db.py @@ -29,6 +29,7 @@ from neutron.callbacks import resources from neutron.common import _deprecate from neutron.common import constants as n_const from neutron.common import utils +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import db_base_plugin_v2 from neutron.db.models import securitygroup as sg_models @@ -256,12 +257,13 @@ class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase): for r in security_group.rules] self._apply_dict_extend_functions(ext_sg.SECURITYGROUPS, res, security_group) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) - def _make_security_group_binding_dict(self, security_group, fields=None): + @staticmethod + def _make_security_group_binding_dict(security_group, fields=None): res = {'port_id': security_group['port_id'], 'security_group_id': security_group['security_group_id']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) @db_api.retry_if_session_inactive() def _create_port_security_group_binding(self, context, port_id, @@ -479,7 +481,7 @@ class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase): self._apply_dict_extend_functions(ext_sg.SECURITYGROUPRULES, res, security_group_rule) - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _make_security_group_rule_filter_dict(self, security_group_rule): sgr = security_group_rule['security_group_rule'] diff --git a/neutron/objects/rbac_db.py b/neutron/objects/rbac_db.py index cc625b244f5..5f4bac52136 100644 --- a/neutron/objects/rbac_db.py +++ b/neutron/objects/rbac_db.py @@ -24,8 +24,8 @@ from neutron._i18n import _ from neutron.callbacks import events from neutron.callbacks import registry from neutron.common import exceptions as n_exc +from neutron.db import _utils as db_utils from neutron.db import api as db_api -from neutron.db import common_db_mixin from neutron.db import rbac_db_mixin from neutron.db import rbac_db_models as models from neutron.extensions import rbac as ext_rbac @@ -68,7 +68,7 @@ class RbacNeutronDbObjectMixin(rbac_db_mixin.RbacPluginMixin, def get_shared_with_tenant(context, rbac_db_model, obj_id, tenant_id): # NOTE(korzen) This method enables to query within already started # session - return (common_db_mixin.model_query(context, rbac_db_model).filter( + return (db_utils.model_query(context, rbac_db_model).filter( and_(rbac_db_model.object_id == obj_id, rbac_db_model.action == models.ACCESS_SHARED, rbac_db_model.target_tenant.in_( @@ -92,7 +92,7 @@ class RbacNeutronDbObjectMixin(rbac_db_mixin.RbacPluginMixin, @classmethod def _get_db_obj_rbac_entries(cls, context, rbac_obj_id, rbac_action): rbac_db_model = cls.rbac_db_model - return common_db_mixin.model_query(context, rbac_db_model).filter( + return db_utils.model_query(context, rbac_db_model).filter( and_(rbac_db_model.object_id == rbac_obj_id, rbac_db_model.action == rbac_action)) diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py index e5858aeb0b2..be5d408ca35 100644 --- a/neutron/plugins/ml2/plugin.py +++ b/neutron/plugins/ml2/plugin.py @@ -45,6 +45,7 @@ from neutron.common import ipv6_utils from neutron.common import rpc as n_rpc from neutron.common import topics from neutron.common import utils +from neutron.db import _utils as db_utils from neutron.db import address_scope_db from neutron.db import agents_db from neutron.db import agentschedulers_db @@ -844,7 +845,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, self.type_manager.extend_network_dict_provider(context, result) result[api.MTU] = self._get_network_mtu(result) - return self._fields(result, fields) + return db_utils.resource_fields(result, fields) @db_api.retry_if_session_inactive() def get_networks(self, context, filters=None, fields=None, @@ -861,7 +862,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, for net in nets: net[api.MTU] = self._get_network_mtu(net) - return [self._fields(net, fields) for net in nets] + return [db_utils.resource_fields(net, fields) for net in nets] def _delete_ports(self, context, port_ids): for port_id in port_ids: diff --git a/neutron/services/auto_allocate/db.py b/neutron/services/auto_allocate/db.py index e534b353bd6..9cab246a9a4 100644 --- a/neutron/services/auto_allocate/db.py +++ b/neutron/services/auto_allocate/db.py @@ -25,6 +25,7 @@ from neutron.callbacks import events from neutron.callbacks import registry from neutron.callbacks import resources from neutron.common import exceptions as c_exc +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import db_base_plugin_v2 @@ -208,13 +209,14 @@ class AutoAllocatedTopologyMixin(common_db_mixin.CommonDbMixin): if network: return network['network_id'] - def _response(self, network_id, tenant_id, fields=None): + @staticmethod + def _response(network_id, tenant_id, fields=None): """Build response for auto-allocated network.""" res = { 'id': network_id, 'tenant_id': tenant_id } - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _get_default_external_network(self, context): """Get the default external network for the deployment.""" diff --git a/neutron/services/provider_configuration.py b/neutron/services/provider_configuration.py index ff1d68b875a..b9322c57c3d 100644 --- a/neutron/services/provider_configuration.py +++ b/neutron/services/provider_configuration.py @@ -26,6 +26,7 @@ import stevedore from neutron._i18n import _, _LW from neutron.api.v2 import attributes as attr +from neutron.db import _utils as db_utils LOG = logging.getLogger(__name__) @@ -265,17 +266,11 @@ class ProviderConfiguration(object): return False return True - def _fields(self, resource, fields): - if fields: - return dict(((key, item) for key, item in resource.items() - if key in fields)) - return resource - def get_service_providers(self, filters=None, fields=None): - return [self._fields({'service_type': k[0], - 'name': k[1], - 'driver': v['driver'], - 'default': v['default']}, - fields) + return [db_utils.resource_fields({'service_type': k[0], + 'name': k[1], + 'driver': v['driver'], + 'default': v['default']}, + fields) for k, v in self.providers.items() if self._check_entry(k, v, filters)] diff --git a/neutron/services/segments/db.py b/neutron/services/segments/db.py index ff34af80690..2cfec2c57a9 100644 --- a/neutron/services/segments/db.py +++ b/neutron/services/segments/db.py @@ -28,6 +28,7 @@ from neutron.callbacks import events from neutron.callbacks import registry from neutron.callbacks import resources from neutron.common import _deprecate +from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db.models import segment as segment_model @@ -43,7 +44,8 @@ _deprecate._moved_global('SegmentHostMapping', new_module=segment_model) class SegmentDbMixin(common_db_mixin.CommonDbMixin): """Mixin class to add segment.""" - def _make_segment_dict(self, segment_db, fields=None): + @staticmethod + def _make_segment_dict(segment_db, fields=None): res = {'id': segment_db['id'], 'network_id': segment_db['network_id'], 'name': segment_db['name'], @@ -54,7 +56,7 @@ class SegmentDbMixin(common_db_mixin.CommonDbMixin): 'hosts': [mapping.host for mapping in segment_db.segment_host_mapping], 'segment_index': segment_db['segment_index']} - return self._fields(res, fields) + return db_utils.resource_fields(res, fields) def _get_segment(self, context, segment_id): try: diff --git a/neutron/tests/unit/db/test_common_db_mixin.py b/neutron/tests/unit/db/test_common_db_mixin.py index 07cb780305e..b35fc9754ba 100644 --- a/neutron/tests/unit/db/test_common_db_mixin.py +++ b/neutron/tests/unit/db/test_common_db_mixin.py @@ -16,7 +16,7 @@ import mock from neutron import context -from neutron.db import common_db_mixin +from neutron.db import _utils as db_utils from neutron.tests.unit import testlib_api @@ -32,7 +32,7 @@ class TestCommonHelpFunctions(testlib_api.SqlTestCase): tx_check = lambda i: setattr(self, '_active', self.admin_ctx.session.is_active) delete_fn = mock.Mock(side_effect=tx_check) - self.assertRaises(ValueError, common_db_mixin.safe_creation, + self.assertRaises(ValueError, db_utils.safe_creation, self.admin_ctx, create_fn, delete_fn, create_bindings) delete_fn.assert_called_once_with(1234) @@ -42,7 +42,7 @@ class TestCommonHelpFunctions(testlib_api.SqlTestCase): create_fn = mock.Mock(return_value={'id': 1234}) create_bindings = mock.Mock(side_effect=ValueError) delete_fn = mock.Mock(side_effect=EnvironmentError) - self.assertRaises(ValueError, common_db_mixin.safe_creation, + self.assertRaises(ValueError, db_utils.safe_creation, self.admin_ctx, create_fn, delete_fn, create_bindings) delete_fn.assert_called_once_with(1234) diff --git a/neutron/tests/unit/extensions/test_portsecurity.py b/neutron/tests/unit/extensions/test_portsecurity.py index 74003f4033d..fac36ca073c 100644 --- a/neutron/tests/unit/extensions/test_portsecurity.py +++ b/neutron/tests/unit/extensions/test_portsecurity.py @@ -17,6 +17,7 @@ from neutron_lib.api import validators from webob import exc from neutron import context +from neutron.db import _utils as db_utils from neutron.db import db_base_plugin_v2 from neutron.db import portsecurity_db from neutron.db import securitygroups_db @@ -83,7 +84,7 @@ class PortSecurityTestPlugin(db_base_plugin_v2.NeutronDbPluginV2, with context.session.begin(subtransactions=True): net = super(PortSecurityTestPlugin, self).get_network( context, id) - return self._fields(net, fields) + return db_utils.resource_fields(net, fields) def create_port(self, context, port): p = port['port']