openstack/neutron
Code Issues Proposed changes

remove the neutron.db._model_query module

Browse Source 
The functionality from model_query is already in neutron-lib and
consumers are using it. This patch removes the _model_query module
from neutron and updates all imports to use neutron-lib's version of
it instead.

NeutronLibImpact

Change-Id: Ib2eae9edb009a93e60b3b0d63ca365056138566b
This commit is contained in:
Boden R
2019-01-02 15:29:17 -07:00
parent 56dfd7291d
commit 1382bf9a32
17 changed files with 28 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
neutron/db/_model_query.py
View File

@ -1,214 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
NOTE: This module shall not be used by external projects. It will be moved
to neutron-lib in due course, and then it can be used from there.
"""
from neutron_lib.api import attributes
from neutron_lib.db import model_query
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as n_exc
from neutron_lib.objects import utils as obj_utils
from neutron_lib.utils import helpers
from oslo_db.sqlalchemy import utils as sa_utils
from sqlalchemy import sql, or_, and_
from sqlalchemy.ext import associationproxy
from neutron._i18n import _
# TODO(boden): remove shims
_model_query_hooks = model_query._model_query_hooks
register_hook = model_query.register_hook
get_hooks = model_query.get_hooks
def query_with_hooks(context, model, field=None):
if field:
if hasattr(model, field):
field = getattr(model, field)
else:
msg = _("'%s' is not supported as field") % field
raise n_exc.InvalidInput(error_message=msg)
query = context.session.query(field)
else:
query = context.session.query(model)
# define basic filter condition for model query
query_filter = None
if db_utils.model_query_scope_is_project(context, model):
if hasattr(model, 'rbac_entries'):
query = query.outerjoin(model.rbac_entries)
rbac_model = model.rbac_entries.property.mapper.class_
query_filter = (
(model.tenant_id == context.tenant_id) |
((rbac_model.action == 'access_as_shared') &
((rbac_model.target_tenant == context.tenant_id) |
(rbac_model.target_tenant == '*'))))
elif hasattr(model, 'shared'):
query_filter = ((model.tenant_id == context.tenant_id) |
(model.shared == sql.true()))
else:
query_filter = (model.tenant_id == context.tenant_id)
# Execute query hooks registered from mixins and plugins
for hook in get_hooks(model):
query_hook = helpers.resolve_ref(hook.get('query'))
if query_hook:
query = query_hook(context, model, query)
filter_hook = helpers.resolve_ref(hook.get('filter'))
if filter_hook:
query_filter = filter_hook(context, model, query_filter)
# NOTE(salvatore-orlando): 'if query_filter' will try to evaluate the
# condition, raising an exception
if query_filter is not None:
query = query.filter(query_filter)
return query
def get_by_id(context, model, object_id):
query = query_with_hooks(context=context, model=model)
return query.filter(model.id == object_id).one()
def apply_filters(query, model, filters, context=None):
if filters:
for key, value in filters.items():
column = getattr(model, key, None)
# NOTE(kevinbenton): if column is a hybrid property that
# references another expression, attempting to convert to
# a boolean will fail so we must compare to None.
# See "An Important Expression Language Gotcha" in:
# docs.sqlalchemy.org/en/rel_0_9/changelog/migration_06.html
if column is not None:
if not value:
query = query.filter(sql.false())
return query
if isinstance(column, associationproxy.AssociationProxy):
# association proxies don't support in_ so we have to
# do multiple equals matches
query = query.filter(
or_(*[column == v for v in value]))
elif isinstance(value, obj_utils.FilterObj):
query = query.filter(value.filter(column))
elif None in value:
# in_() operator does not support NULL element so we have
# to do multiple equals matches
query = query.filter(
or_(*[column == v for v in value]))
else:
query = query.filter(column.in_(value))
elif key == 'shared' and hasattr(model, 'rbac_entries'):
# translate a filter on shared into a query against the
# object's rbac entries
rbac = model.rbac_entries.property.mapper.class_
matches = [rbac.target_tenant == '*']
if context:
matches.append(rbac.target_tenant == context.tenant_id)
# any 'access_as_shared' records that match the
# wildcard or requesting tenant
is_shared = and_(rbac.action == 'access_as_shared',
or_(*matches))
if not value[0]:
# NOTE(kevinbenton): we need to find objects that don't
# have an entry that matches the criteria above so
# we use a subquery to exclude them.
# We can't just filter the inverse of the query above
# because that will still give us a network shared to
# our tenant (or wildcard) if it's shared to another
# tenant.
# This is the column joining the table to rbac via
# the object_id. We can't just use model.id because
# subnets join on network.id so we have to inspect the
# relationship.
join_cols = model.rbac_entries.property.local_columns
oid_col = list(join_cols)[0]
is_shared = ~oid_col.in_(
query.session.query(rbac.object_id).filter(is_shared)
)
elif (not context or
not db_utils.model_query_scope_is_project(
context, model)):
# we only want to join if we aren't using the subquery
# and if we aren't already joined because this is a
# scoped query
query = query.outerjoin(model.rbac_entries)
query = query.filter(is_shared)
for hook in get_hooks(model):
result_filter = helpers.resolve_ref(
hook.get('result_filters', None))
if result_filter:
query = result_filter(query, filters)
return query
def get_collection_query(context, model, filters=None, sorts=None, limit=None,
marker_obj=None, page_reverse=False):
collection = query_with_hooks(context, model)
collection = apply_filters(collection, model, filters, context)
if sorts:
sort_keys = db_utils.get_and_validate_sort_keys(sorts, model)
sort_dirs = db_utils.get_sort_dirs(sorts, page_reverse)
# we always want deterministic results for sorted queries
# so add unique keys to limit queries when present.
# (http://docs.sqlalchemy.org/en/latest/orm/
# loading_relationships.html#subqueryload-ordering)
# (http://docs.sqlalchemy.org/en/latest/faq/
# ormconfiguration.html#faq-subqueryload-limit-sort)
for k in _unique_keys(model):
if k not in sort_keys:
sort_keys.append(k)
sort_dirs.append('asc')
collection = sa_utils.paginate_query(collection, model, limit,
marker=marker_obj,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
return collection
def _unique_keys(model):
# just grab first set of unique keys and use them.
# if model has no unqiue sets, 'paginate_query' will
# warn if sorting is unstable
uk_sets = sa_utils.get_unique_keys(model)
return uk_sets[0] if uk_sets else []
def get_collection(context, model, dict_func,
filters=None, fields=None,
sorts=None, limit=None, marker_obj=None,
page_reverse=False):
query = get_collection_query(context, model,
filters=filters, sorts=sorts,
limit=limit, marker_obj=marker_obj,
page_reverse=page_reverse)
items = [
attributes.populate_project_info(
dict_func(c, fields) if dict_func else c)
for c in query
]
if limit and page_reverse:
items.reverse()
return items
def get_values(context, model, field, filters=None):
query = query_with_hooks(context, model, field=field)
query = apply_filters(query, model, filters, context)
return [c[0] for c in query]
def get_collection_count(context, model, filters=None):
return get_collection_query(context, model, filters).count()

2
neutron/db/agents_db.py
View File

@ -27,6 +27,7 @@ from neutron_lib.callbacks import resources
from neutron_lib import constants
from neutron_lib import context
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import utils as db_utils
from neutron_lib.exceptions import agent as agent_exc
from neutron_lib.exceptions import availability_zone as az_exc
@ -43,7 +44,6 @@ from neutron.agent.common import utils
from neutron.api.rpc.callbacks import version_manager
from neutron.common import constants as n_const
from neutron.conf.agent.database import agents_db
from neutron.db import _model_query as model_query
from neutron.db.models import agent as agent_model
from neutron.extensions import _availability_zone_filter_lib as azfil_ext
from neutron.extensions import agent as ext_agent

27
neutron/db/common_db_mixin.py
View File

@ -13,11 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as db_utils
from neutron.db import _model_query
# TODO(HenryG): Deprecate and schedule for removal
class CommonDbMixin(object):
@ -29,7 +28,7 @@ class CommonDbMixin(object):
@staticmethod
def _model_query(context, model):
return _model_query.query_with_hooks(context, model)
return model_query.query_with_hooks(context, model)
@staticmethod
def _fields(resource, fields):
@ -37,35 +36,35 @@ class CommonDbMixin(object):
@staticmethod
def _get_by_id(context, model, id):
return _model_query.get_by_id(context, model, id)
return model_query.get_by_id(context, model, id)
@staticmethod
def _apply_filters_to_query(query, model, filters, context=None):
return _model_query.apply_filters(query, model, filters, context)
return model_query.apply_filters(query, model, filters, context)
@staticmethod
def _get_collection_query(context, model,
filters=None, sorts=None,
limit=None, marker_obj=None,
page_reverse=False):
return _model_query.get_collection_query(context, model,
filters, sorts,
limit, marker_obj,
page_reverse)
return model_query.get_collection_query(context, model,
filters, sorts,
limit, marker_obj,
page_reverse)
@staticmethod
def _get_collection(context, model, dict_func,
filters=None, fields=None, sorts=None,
limit=None, marker_obj=None,
page_reverse=False):
return _model_query.get_collection(context, model, dict_func,
filters, fields, sorts,
limit, marker_obj,
page_reverse)
return model_query.get_collection(context, model, dict_func,
filters, fields, sorts,
limit, marker_obj,
page_reverse)
@staticmethod
def _get_collection_count(context, model, filters=None):
return _model_query.get_collection_count(context, model, filters)
return model_query.get_collection_count(context, model, filters)
# TODO(HenryG): Remove this when available in neutron-lib
def _get_marker_obj(self, context, resource, limit, marker):

2
neutron/db/db_base_plugin_common.py
View File

@ -24,6 +24,7 @@ from neutron_lib.api.definitions import subnetpool as subnetpool_def
from neutron_lib.api import validators
from neutron_lib import constants
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as n_exc
@ -34,7 +35,6 @@ from sqlalchemy.orm import exc
from neutron.common import constants as n_const
from neutron.common import exceptions
from neutron.db import _model_query as model_query
from neutron.db import common_db_mixin
from neutron.db import models_v2
from neutron.objects import base as base_obj

2
neutron/db/db_base_plugin_v2.py
View File

@ -27,6 +27,7 @@ from neutron_lib.callbacks import resources
from neutron_lib import constants
from neutron_lib import context as ctx
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as ndb_utils
from neutron_lib import exceptions as exc
@ -46,7 +47,6 @@ from neutron.api.rpc.agentnotifiers import l3_rpc_agent_api
from neutron.common import exceptions as n_exc
from neutron.common import ipv6_utils
from neutron.common import utils
from neutron.db import _model_query as model_query
from neutron.db import db_base_plugin_common
from neutron.db import ipam_pluggable_backend
from neutron.db import models_v2

2
neutron/db/external_net_db.py
View File

@ -20,6 +20,7 @@ from neutron_lib.callbacks import events
from neutron_lib.callbacks import registry
from neutron_lib.callbacks import resources
from neutron_lib import constants
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as n_exc
@ -29,7 +30,6 @@ from neutron_lib.plugins import directory
from sqlalchemy.sql import expression as expr
from neutron._i18n import _
from neutron.db import _model_query as model_query
from neutron.db import models_v2
from neutron.extensions import rbac as rbac_ext
from neutron.objects import network as net_obj

2
neutron/db/l3_db.py
View File

@ -27,6 +27,7 @@ from neutron_lib.callbacks import resources
from neutron_lib import constants
from neutron_lib import context as n_ctx
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as lib_db_utils
from neutron_lib import exceptions as n_exc
@ -45,7 +46,6 @@ from neutron.api.rpc.agentnotifiers import l3_rpc_agent_api
from neutron.common import ipv6_utils
from neutron.common import rpc as n_rpc
from neutron.common import utils
from neutron.db import _model_query as model_query
from neutron.db import _utils as db_utils
from neutron.db.models import l3 as l3_models
from neutron.db import models_v2

2
neutron/db/portbindings_db.py
View File

@ -17,10 +17,10 @@ from neutron_lib.api.definitions import port as port_def
from neutron_lib.api.definitions import portbindings
from neutron_lib.api import validators
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.plugins import directory
from neutron.db import _model_query as model_query
from neutron.db.models import portbinding as pmodels
from neutron.db import models_v2
from neutron.db import portbindings_base

2
neutron/db/securitygroups_db.py
View File

@ -21,6 +21,7 @@ from neutron_lib.callbacks import registry
from neutron_lib.callbacks import resources
from neutron_lib import constants
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as n_exc
@ -33,7 +34,6 @@ from sqlalchemy.orm import scoped_session
from neutron._i18n import _
from neutron.common import constants as n_const
from neutron.common import utils
from neutron.db import _model_query as model_query
from neutron.db.models import securitygroup as sg_models
from neutron.extensions import securitygroup as ext_sg
from neutron.objects import base as base_obj

3
neutron/objects/db/api.py
View File

@ -13,12 +13,11 @@
# TODO(ihrachys): cover the module with functional tests targeting supported
# backends
from neutron_lib.db import model_query
from neutron_lib import exceptions as n_exc
from neutron_lib.objects import utils as obj_utils
from oslo_utils import uuidutils
from neutron.db import _model_query as model_query
# Common database operation implementations
def _get_filter_query(obj_cls, context, **kwargs):

2
neutron/objects/subnet.py
View File

@ -13,12 +13,12 @@
import netaddr
from neutron_lib.api import validators
from neutron_lib import constants as const
from neutron_lib.db import model_query
from oslo_versionedobjects import fields as obj_fields
from sqlalchemy import and_, or_
from neutron.common import utils
from neutron.db import _model_query as model_query
from neutron.db.models import segment as segment_model
from neutron.db.models import subnet_service_type
from neutron.db import models_v2

2
neutron/objects/tag.py
View File

@ -12,11 +12,11 @@
import functools
from neutron_lib.db import model_query
from sqlalchemy.orm import aliased
from oslo_versionedobjects import fields as obj_fields
from neutron.db import _model_query as model_query
from neutron.db.models import tag as tag_model
from neutron.db import standard_attr
from neutron.objects import base

2
neutron/plugins/ml2/plugin.py
View File

@ -36,6 +36,7 @@ from neutron_lib.callbacks import registry
from neutron_lib.callbacks import resources
from neutron_lib import constants as const
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as exc
@ -70,7 +71,6 @@ from neutron.common import constants as n_const
from neutron.common import exceptions as n_exc
from neutron.common import rpc as n_rpc
from neutron.common import utils
from neutron.db import _model_query as model_query
from neutron.db import address_scope_db
from neutron.db import agents_db
from neutron.db import agentschedulers_db

2
neutron/services/tag/tag_plugin.py
View File

@ -13,13 +13,13 @@
#
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib.objects import exceptions as obj_exc
from neutron_lib.plugins import directory
from oslo_log import helpers as log_helpers
from sqlalchemy.orm import exc
from neutron.db import _model_query as model_query
from neutron.db import common_db_mixin
from neutron.db import standard_attr
from neutron.extensions import tagging

2
neutron/services/timestamp/timestamp_db.py
View File

@ -13,12 +13,12 @@
# under the License.
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib.db import resource_extend
from neutron_lib import exceptions as n_exc
from oslo_utils import timeutils
from sqlalchemy.orm import session as se
from neutron.db import _model_query as model_query
from neutron.db import standard_attr
CHANGED_SINCE = 'changed_since'

2
neutron/tests/unit/objects/db/test_api.py
View File

@ -14,10 +14,10 @@ import copy
import mock
from neutron_lib import context
from neutron_lib.db import model_query
from neutron_lib import exceptions as n_exc
from neutron_lib.objects import utils as obj_utils
from neutron.db import _model_query as model_query
from neutron.objects import base
from neutron.objects.db import api
from neutron.objects import network

2
neutron/tests/unit/objects/test_base.py
View File

@ -21,6 +21,7 @@ import netaddr
from neutron_lib import constants
from neutron_lib import context
from neutron_lib.db import api as db_api
from neutron_lib.db import model_query
from neutron_lib import exceptions as n_exc
from neutron_lib.objects import exceptions as o_exc
from neutron_lib.objects import utils as obj_utils
@ -33,7 +34,6 @@ from oslo_versionedobjects import exception
from oslo_versionedobjects import fields as obj_fields
import testtools
from neutron.db import _model_query as model_query
from neutron import objects
from neutron.objects import agent
from neutron.objects import base