Refactor CommonDbMixin for removal
Move the CommonDbMixin methods to model_query.py and resource_extend.py This leaves CommonDbMixin as just a shim, and it should be easy to remove once subprojects have stopped depending on it. Related-Blueprint: neutron-lib Change-Id: I5b804e09e630d88d551271d9731cc1f65c065259
This commit is contained in:
Henry Gessau
parent
b3c0d5f239
commit
ef471a652b
|
|
@ -837,3 +837,10 @@ except AttributeError:
|
|||
def make_weak_ref(f):
|
||||
"""Make a weak reference to a function accounting for bound methods."""
|
||||
return weak_method(f) if hasattr(f, '__self__') else weakref.ref(f)
|
||||
|
||||
|
||||
def resolve_ref(ref):
|
||||
"""Handles dereference of weakref."""
|
||||
if isinstance(ref, weakref.ref):
|
||||
ref = ref()
|
||||
return ref
|
||||
|
|
|
|||
|
|
@ -16,7 +16,14 @@ NOTE: This module shall not be used by external projects. It will be moved
|
|||
to neutron-lib in due course, and then it can be used from there.
|
||||
"""
|
||||
|
||||
from neutron_lib.db import utils as db_utils
|
||||
from oslo_db.sqlalchemy import utils as sa_utils
|
||||
from sqlalchemy import sql, or_, and_
|
||||
from sqlalchemy.ext import associationproxy
|
||||
|
||||
from neutron.api.v2 import attributes
|
||||
from neutron.common import utils
|
||||
from neutron.db import _utils as ndb_utils
|
||||
|
||||
# Classes implementing extensions will register hooks into this dictionary
|
||||
# for "augmenting" the "core way" of building a query for retrieving objects
|
||||
|
|
@ -123,3 +130,160 @@ def get_hooks(model):
|
|||
|
||||
"""
|
||||
return _model_query_hooks.get(model, {}).values()
|
||||
|
||||
|
||||
def query_with_hooks(context, model):
|
||||
query = context.session.query(model)
|
||||
# define basic filter condition for model query
|
||||
query_filter = None
|
||||
if ndb_utils.model_query_scope_is_project(context, model):
|
||||
if hasattr(model, 'rbac_entries'):
|
||||
query = query.outerjoin(model.rbac_entries)
|
||||
rbac_model = model.rbac_entries.property.mapper.class_
|
||||
query_filter = (
|
||||
(model.tenant_id == context.tenant_id) |
|
||||
((rbac_model.action == 'access_as_shared') &
|
||||
((rbac_model.target_tenant == context.tenant_id) |
|
||||
(rbac_model.target_tenant == '*'))))
|
||||
elif hasattr(model, 'shared'):
|
||||
query_filter = ((model.tenant_id == context.tenant_id) |
|
||||
(model.shared == sql.true()))
|
||||
else:
|
||||
query_filter = (model.tenant_id == context.tenant_id)
|
||||
# Execute query hooks registered from mixins and plugins
|
||||
for hook in get_hooks(model):
|
||||
query_hook = utils.resolve_ref(hook.get('query'))
|
||||
if query_hook:
|
||||
query = query_hook(context, model, query)
|
||||
|
||||
filter_hook = utils.resolve_ref(hook.get('filter'))
|
||||
if filter_hook:
|
||||
query_filter = filter_hook(context, model, query_filter)
|
||||
|
||||
# NOTE(salvatore-orlando): 'if query_filter' will try to evaluate the
|
||||
# condition, raising an exception
|
||||
if query_filter is not None:
|
||||
query = query.filter(query_filter)
|
||||
return query
|
||||
|
||||
|
||||
def get_by_id(context, model, object_id):
|
||||
query = query_with_hooks(context=context, model=model)
|
||||
return query.filter(model.id == object_id).one()
|
||||
|
||||
|
||||
def apply_filters(query, model, filters, context=None):
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
column = getattr(model, key, None)
|
||||
# NOTE(kevinbenton): if column is a hybrid property that
|
||||
# references another expression, attempting to convert to
|
||||
# a boolean will fail so we must compare to None.
|
||||
# See "An Important Expression Language Gotcha" in:
|
||||
# docs.sqlalchemy.org/en/rel_0_9/changelog/migration_06.html
|
||||
if column is not None:
|
||||
if not value:
|
||||
query = query.filter(sql.false())
|
||||
return query
|
||||
if isinstance(column, associationproxy.AssociationProxy):
|
||||
# association proxies don't support in_ so we have to
|
||||
# do multiple equals matches
|
||||
query = query.filter(
|
||||
or_(*[column == v for v in value]))
|
||||
else:
|
||||
query = query.filter(column.in_(value))
|
||||
elif key == 'shared' and hasattr(model, 'rbac_entries'):
|
||||
# translate a filter on shared into a query against the
|
||||
# object's rbac entries
|
||||
rbac = model.rbac_entries.property.mapper.class_
|
||||
matches = [rbac.target_tenant == '*']
|
||||
if context:
|
||||
matches.append(rbac.target_tenant == context.tenant_id)
|
||||
# any 'access_as_shared' records that match the
|
||||
# wildcard or requesting tenant
|
||||
is_shared = and_(rbac.action == 'access_as_shared',
|
||||
or_(*matches))
|
||||
if not value[0]:
|
||||
# NOTE(kevinbenton): we need to find objects that don't
|
||||
# have an entry that matches the criteria above so
|
||||
# we use a subquery to exclude them.
|
||||
# We can't just filter the inverse of the query above
|
||||
# because that will still give us a network shared to
|
||||
# our tenant (or wildcard) if it's shared to another
|
||||
# tenant.
|
||||
# This is the column joining the table to rbac via
|
||||
# the object_id. We can't just use model.id because
|
||||
# subnets join on network.id so we have to inspect the
|
||||
# relationship.
|
||||
join_cols = model.rbac_entries.property.local_columns
|
||||
oid_col = list(join_cols)[0]
|
||||
is_shared = ~oid_col.in_(
|
||||
query.session.query(rbac.object_id).filter(is_shared)
|
||||
)
|
||||
elif (not context or
|
||||
not ndb_utils.model_query_scope_is_project(context,
|
||||
model)):
|
||||
# we only want to join if we aren't using the subquery
|
||||
# and if we aren't already joined because this is a
|
||||
# scoped query
|
||||
query = query.outerjoin(model.rbac_entries)
|
||||
query = query.filter(is_shared)
|
||||
for hook in get_hooks(model):
|
||||
result_filter = utils.resolve_ref(hook.get('result_filters', None))
|
||||
if result_filter:
|
||||
query = result_filter(query, filters)
|
||||
return query
|
||||
|
||||
|
||||
def get_collection_query(context, model, filters=None, sorts=None, limit=None,
|
||||
marker_obj=None, page_reverse=False):
|
||||
collection = query_with_hooks(context, model)
|
||||
collection = apply_filters(collection, model, filters, context)
|
||||
if sorts:
|
||||
sort_keys = db_utils.get_and_validate_sort_keys(sorts, model)
|
||||
sort_dirs = db_utils.get_sort_dirs(sorts, page_reverse)
|
||||
# we always want deterministic results for sorted queries
|
||||
# so add unique keys to limit queries when present.
|
||||
# (http://docs.sqlalchemy.org/en/latest/orm/
|
||||
# loading_relationships.html#subqueryload-ordering)
|
||||
# (http://docs.sqlalchemy.org/en/latest/faq/
|
||||
# ormconfiguration.html#faq-subqueryload-limit-sort)
|
||||
for k in _unique_keys(model):
|
||||
if k not in sort_keys:
|
||||
sort_keys.append(k)
|
||||
sort_dirs.append('asc')
|
||||
collection = sa_utils.paginate_query(collection, model, limit,
|
||||
marker=marker_obj,
|
||||
sort_keys=sort_keys,
|
||||
sort_dirs=sort_dirs)
|
||||
return collection
|
||||
|
||||
|
||||
def _unique_keys(model):
|
||||
# just grab first set of unique keys and use them.
|
||||
# if model has no unqiue sets, 'paginate_query' will
|
||||
# warn if sorting is unstable
|
||||
uk_sets = sa_utils.get_unique_keys(model)
|
||||
return uk_sets[0] if uk_sets else []
|
||||
|
||||
|
||||
def get_collection(context, model, dict_func,
|
||||
filters=None, fields=None,
|
||||
sorts=None, limit=None, marker_obj=None,
|
||||
page_reverse=False):
|
||||
query = get_collection_query(context, model,
|
||||
filters=filters, sorts=sorts,
|
||||
limit=limit, marker_obj=marker_obj,
|
||||
page_reverse=page_reverse)
|
||||
items = [
|
||||
attributes.populate_project_info(
|
||||
dict_func(c, fields) if dict_func else c)
|
||||
for c in query
|
||||
]
|
||||
if limit and page_reverse:
|
||||
items.reverse()
|
||||
return items
|
||||
|
||||
|
||||
def get_collection_count(context, model, filters=None):
|
||||
return get_collection_query(context, model, filters).count()
|
||||
|
|
|
|||
|
|
@ -68,6 +68,13 @@ def get_funcs(resource):
|
|||
return _resource_extend_functions.get(resource, [])
|
||||
|
||||
|
||||
def apply_funcs(resource_type, response, db_object):
|
||||
for func in get_funcs(resource_type):
|
||||
resolved_func = utils.resolve_ref(func)
|
||||
if resolved_func:
|
||||
resolved_func(response, db_object)
|
||||
|
||||
|
||||
def extends(resources):
|
||||
"""Use to decorate methods on classes before initialization.
|
||||
|
||||
|
|
|
|||
|
|
@ -15,14 +15,6 @@
|
|||
|
||||
import weakref
|
||||
|
||||
from neutron_lib.db import utils as db_utils
|
||||
from oslo_db.sqlalchemy import utils as sa_utils
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy.ext import associationproxy
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy import sql
|
||||
|
||||
from neutron.api.v2 import attributes
|
||||
from neutron.db import _model_query
|
||||
from neutron.db import _resource_extend
|
||||
from neutron.db import _utils as ndb_utils
|
||||
|
|
@ -35,15 +27,9 @@ model_query = ndb_utils.model_query
|
|||
resource_fields = ndb_utils.resource_fields
|
||||
|
||||
|
||||
def _resolve_ref(ref):
|
||||
"""Handles dereference of weakref."""
|
||||
if isinstance(ref, weakref.ref):
|
||||
ref = ref()
|
||||
return ref
|
||||
|
||||
|
||||
# TODO(HenryG): Deprecate and schedule for removal
|
||||
class CommonDbMixin(object):
|
||||
"""Common methods used in core and service plugins."""
|
||||
"""Deprecated."""
|
||||
|
||||
@staticmethod
|
||||
def register_model_query_hook(model, name, query_hook, filter_hook,
|
||||
|
|
@ -59,188 +45,61 @@ class CommonDbMixin(object):
|
|||
@property
|
||||
# TODO(HenryG): Remove; used only by vmware-nsx.
|
||||
def safe_reference(self):
|
||||
"""Return a weakref to the instance.
|
||||
|
||||
Minimize the potential for the instance persisting
|
||||
unnecessarily in memory by returning a weakref proxy that
|
||||
won't prevent deallocation.
|
||||
"""
|
||||
return weakref.proxy(self)
|
||||
|
||||
# TODO(HenryG): Remove this when available in neutron-lib
|
||||
def model_query_scope(self, context, model):
|
||||
@staticmethod
|
||||
def model_query_scope(context, model):
|
||||
return ndb_utils.model_query_scope_is_project(context, model)
|
||||
|
||||
def _model_query(self, context, model):
|
||||
query = context.session.query(model)
|
||||
# define basic filter condition for model query
|
||||
query_filter = None
|
||||
if ndb_utils.model_query_scope_is_project(context, model):
|
||||
if hasattr(model, 'rbac_entries'):
|
||||
query = query.outerjoin(model.rbac_entries)
|
||||
rbac_model = model.rbac_entries.property.mapper.class_
|
||||
query_filter = (
|
||||
(model.tenant_id == context.tenant_id) |
|
||||
((rbac_model.action == 'access_as_shared') &
|
||||
((rbac_model.target_tenant == context.tenant_id) |
|
||||
(rbac_model.target_tenant == '*'))))
|
||||
elif hasattr(model, 'shared'):
|
||||
query_filter = ((model.tenant_id == context.tenant_id) |
|
||||
(model.shared == sql.true()))
|
||||
else:
|
||||
query_filter = (model.tenant_id == context.tenant_id)
|
||||
# Execute query hooks registered from mixins and plugins
|
||||
for hook in _model_query.get_hooks(model):
|
||||
query_hook = _resolve_ref(hook.get('query'))
|
||||
if query_hook:
|
||||
query = query_hook(context, model, query)
|
||||
|
||||
filter_hook = _resolve_ref(hook.get('filter'))
|
||||
if filter_hook:
|
||||
query_filter = filter_hook(context, model, query_filter)
|
||||
|
||||
# NOTE(salvatore-orlando): 'if query_filter' will try to evaluate the
|
||||
# condition, raising an exception
|
||||
if query_filter is not None:
|
||||
query = query.filter(query_filter)
|
||||
return query
|
||||
|
||||
# TODO(HenryG): Remove this when available in neutron-lib
|
||||
def _fields(self, resource, fields):
|
||||
return ndb_utils.resource_fields(resource, fields)
|
||||
|
||||
def _get_by_id(self, context, model, id):
|
||||
query = self._model_query(context, model)
|
||||
return query.filter(model.id == id).one()
|
||||
|
||||
def _apply_filters_to_query(self, query, model, filters, context=None):
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
column = getattr(model, key, None)
|
||||
# NOTE(kevinbenton): if column is a hybrid property that
|
||||
# references another expression, attempting to convert to
|
||||
# a boolean will fail so we must compare to None.
|
||||
# See "An Important Expression Language Gotcha" in:
|
||||
# docs.sqlalchemy.org/en/rel_0_9/changelog/migration_06.html
|
||||
if column is not None:
|
||||
if not value:
|
||||
query = query.filter(sql.false())
|
||||
return query
|
||||
if isinstance(column, associationproxy.AssociationProxy):
|
||||
# association proxies don't support in_ so we have to
|
||||
# do multiple equals matches
|
||||
query = query.filter(
|
||||
or_(*[column == v for v in value]))
|
||||
else:
|
||||
query = query.filter(column.in_(value))
|
||||
elif key == 'shared' and hasattr(model, 'rbac_entries'):
|
||||
# translate a filter on shared into a query against the
|
||||
# object's rbac entries
|
||||
rbac = model.rbac_entries.property.mapper.class_
|
||||
matches = [rbac.target_tenant == '*']
|
||||
if context:
|
||||
matches.append(rbac.target_tenant == context.tenant_id)
|
||||
# any 'access_as_shared' records that match the
|
||||
# wildcard or requesting tenant
|
||||
is_shared = and_(rbac.action == 'access_as_shared',
|
||||
or_(*matches))
|
||||
if not value[0]:
|
||||
# NOTE(kevinbenton): we need to find objects that don't
|
||||
# have an entry that matches the criteria above so
|
||||
# we use a subquery to exclude them.
|
||||
# We can't just filter the inverse of the query above
|
||||
# because that will still give us a network shared to
|
||||
# our tenant (or wildcard) if it's shared to another
|
||||
# tenant.
|
||||
# This is the column joining the table to rbac via
|
||||
# the object_id. We can't just use model.id because
|
||||
# subnets join on network.id so we have to inspect the
|
||||
# relationship.
|
||||
join_cols = model.rbac_entries.property.local_columns
|
||||
oid_col = list(join_cols)[0]
|
||||
is_shared = ~oid_col.in_(
|
||||
query.session.query(rbac.object_id).
|
||||
filter(is_shared)
|
||||
)
|
||||
elif (not context or
|
||||
not self.model_query_scope(context, model)):
|
||||
# we only want to join if we aren't using the subquery
|
||||
# and if we aren't already joined because this is a
|
||||
# scoped query
|
||||
query = query.outerjoin(model.rbac_entries)
|
||||
query = query.filter(is_shared)
|
||||
for hook in _model_query.get_hooks(model):
|
||||
result_filter = _resolve_ref(hook.get('result_filters', None))
|
||||
|
||||
if result_filter:
|
||||
query = result_filter(query, filters)
|
||||
return query
|
||||
@staticmethod
|
||||
def _model_query(context, model):
|
||||
return _model_query.query_with_hooks(context, model)
|
||||
|
||||
@staticmethod
|
||||
def _apply_dict_extend_functions(resource_type,
|
||||
response, db_object):
|
||||
for func in _resource_extend.get_funcs(resource_type):
|
||||
resolved_func = _resolve_ref(func)
|
||||
if resolved_func:
|
||||
resolved_func(response, db_object)
|
||||
def _fields(resource, fields):
|
||||
return ndb_utils.resource_fields(resource, fields)
|
||||
|
||||
def _get_collection_query(self, context, model, filters=None,
|
||||
sorts=None, limit=None, marker_obj=None,
|
||||
@staticmethod
|
||||
def _get_by_id(context, model, id):
|
||||
return _model_query.get_by_id(context, model, id)
|
||||
|
||||
@staticmethod
|
||||
def _apply_filters_to_query(query, model, filters, context=None):
|
||||
return _model_query.apply_filters(query, model, filters, context)
|
||||
|
||||
@staticmethod
|
||||
def _apply_dict_extend_functions(resource_type, response, db_object):
|
||||
_resource_extend.apply_funcs(resource_type, response, db_object)
|
||||
|
||||
@staticmethod
|
||||
def _get_collection_query(context, model,
|
||||
filters=None, sorts=None,
|
||||
limit=None, marker_obj=None,
|
||||
page_reverse=False):
|
||||
collection = self._model_query(context, model)
|
||||
collection = self._apply_filters_to_query(collection, model, filters,
|
||||
context)
|
||||
if sorts:
|
||||
sort_keys = db_utils.get_and_validate_sort_keys(sorts, model)
|
||||
sort_dirs = db_utils.get_sort_dirs(sorts, page_reverse)
|
||||
# we always want deterministic results for sorted queries
|
||||
# so add unique keys to limit queries when present.
|
||||
# (http://docs.sqlalchemy.org/en/latest/orm/
|
||||
# loading_relationships.html#subqueryload-ordering)
|
||||
# (http://docs.sqlalchemy.org/en/latest/faq/
|
||||
# ormconfiguration.html#faq-subqueryload-limit-sort)
|
||||
for k in self._unique_keys(model, marker_obj):
|
||||
if k not in sort_keys:
|
||||
sort_keys.append(k)
|
||||
sort_dirs.append('asc')
|
||||
collection = sa_utils.paginate_query(collection, model, limit,
|
||||
marker=marker_obj,
|
||||
sort_keys=sort_keys,
|
||||
sort_dirs=sort_dirs)
|
||||
return collection
|
||||
return _model_query.get_collection_query(context, model,
|
||||
filters, sorts,
|
||||
limit, marker_obj,
|
||||
page_reverse)
|
||||
|
||||
def _unique_keys(self, model, marker_obj):
|
||||
# just grab first set of unique keys and use them.
|
||||
# if model has no unqiue sets, 'paginate_query' will
|
||||
# warn if sorting is unstable
|
||||
uk_sets = sa_utils.get_unique_keys(model)
|
||||
return uk_sets[0] if uk_sets else []
|
||||
|
||||
def _get_collection(self, context, model, dict_func, filters=None,
|
||||
fields=None, sorts=None, limit=None, marker_obj=None,
|
||||
@staticmethod
|
||||
def _get_collection(context, model, dict_func,
|
||||
filters=None, fields=None, sorts=None,
|
||||
limit=None, marker_obj=None,
|
||||
page_reverse=False):
|
||||
query = self._get_collection_query(context, model, filters=filters,
|
||||
sorts=sorts,
|
||||
limit=limit,
|
||||
marker_obj=marker_obj,
|
||||
page_reverse=page_reverse)
|
||||
items = [
|
||||
attributes.populate_project_info(
|
||||
dict_func(c, fields) if dict_func else c)
|
||||
for c in query
|
||||
]
|
||||
if limit and page_reverse:
|
||||
items.reverse()
|
||||
return items
|
||||
return _model_query.get_collection(context, model, dict_func,
|
||||
filters, fields, sorts,
|
||||
limit, marker_obj,
|
||||
page_reverse)
|
||||
|
||||
def _get_collection_count(self, context, model, filters=None):
|
||||
return self._get_collection_query(context, model, filters).count()
|
||||
@staticmethod
|
||||
def _get_collection_count(context, model, filters=None):
|
||||
return _model_query.get_collection_count(context, model, filters)
|
||||
|
||||
def _get_marker_obj(self, context, resource, limit, marker):
|
||||
if limit and marker:
|
||||
return getattr(self, '_get_%s' % resource)(context, marker)
|
||||
return None
|
||||
|
||||
# TODO(HenryG): Remove this when available in neutron-lib
|
||||
def _filter_non_model_columns(self, data, model):
|
||||
@staticmethod
|
||||
def _filter_non_model_columns(data, model):
|
||||
return ndb_utils.filter_non_model_columns(data, model)
|
||||
|
|
|
|||
Loading…
Reference in New Issue