openstack
/
neutron
Code Issues Proposed changes

Use paginate_query from oslo_db 

The paginate_query method was copied from nova which was copied
from glance. Now it is available in oslo_db.

Check and convert the sort keys and sort directions for
consumption by the oslo_db version of the method, and fix up
some grammar in the exception messages.

This work is related to the neutron-lib effort. The lib should
not propagate neutron's copy of paginate_query().

Related-Blueprint: neutron-lib

Change-Id: Ie7da16b94fa2023c9c3d84d96d55f33d0f76903f
This commit is contained in:
Henry Gessau 2016-02-28 20:49:51 -05:00 committed by Henry Gessau
parent b047e3c28a
commit 2f767839c9
3 changed files with 29 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
neutron/db/common_db_mixin.py
View File

@ -16,6 +16,8 @@
import contextlib
import weakref
from neutron_lib.db import utils as db_utils
from oslo_db.sqlalchemy import utils as sa_utils
from oslo_log import log as logging
from oslo_utils import excutils
import six
@ -25,7 +27,6 @@ from sqlalchemy import or_
from sqlalchemy import sql
from neutron._i18n import _LE
from neutron.db import sqlalchemyutils
LOG = logging.getLogger(__name__)
@ -276,11 +277,13 @@ class CommonDbMixin(object):
collection = self._model_query(context, model)
collection = self._apply_filters_to_query(collection, model, filters,
context)
if limit and page_reverse and sorts:
sorts = [(s[0], not s[1]) for s in sorts]
collection = sqlalchemyutils.paginate_query(collection, model, limit,
sorts,
marker_obj=marker_obj)
if sorts:
sort_keys = db_utils.get_and_validate_sort_keys(sorts, model)
sort_dirs = db_utils.get_sort_dirs(sorts, page_reverse)
collection = sa_utils.paginate_query(collection, model, limit,
marker=marker_obj,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
return collection
def _get_collection(self, context, model, dict_func, filters=None,

14
neutron/db/db_base_plugin_v2.py
View File

@ -18,9 +18,11 @@ import functools
import netaddr
from neutron_lib.api import validators
from neutron_lib import constants
from neutron_lib.db import utils as db_utils
from neutron_lib import exceptions as exc
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import utils as sa_utils
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import uuidutils
@ -47,7 +49,6 @@ from neutron.db import ipam_pluggable_backend
from neutron.db import models_v2
from neutron.db import rbac_db_mixin as rbac_mixin
from neutron.db import rbac_db_models as rbac_db
from neutron.db import sqlalchemyutils
from neutron.db import standardattrdescription_db as stattr_db
from neutron.extensions import l3
from neutron import ipam
@ -1366,10 +1367,13 @@ class NeutronDbPluginV2(db_base_plugin_common.DbBasePluginCommon,
query = query.filter(IPAllocation.subnet_id.in_(subnet_ids))
query = self._apply_filters_to_query(query, Port, filters, context)
if limit and page_reverse and sorts:
sorts = [(s[0], not s[1]) for s in sorts]
query = sqlalchemyutils.paginate_query(query, Port, limit,
sorts, marker_obj)
if sorts:
sort_keys = db_utils.get_and_validate_sort_keys(sorts, Port)
sort_dirs = db_utils.get_sort_dirs(sorts, page_reverse)
query = sa_utils.paginate_query(query, Port, limit,
marker=marker_obj,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
return query
def get_ports(self, context, filters=None, fields=None,

94
neutron/db/sqlalchemyutils.py
View File

@ -13,92 +13,20 @@
# License for the specific language governing permissions and limitations
# under the License.
from neutron_lib import exceptions as n_exc
from six import moves
import sqlalchemy
from sqlalchemy.orm import properties
from neutron._i18n import _
from debtcollector import removals
from neutron_lib.db import utils as db_utils
from oslo_db.sqlalchemy import utils as sa_utils
@removals.remove(version='newton', removal_version='ocata',
message='Use ' + sa_utils.__name__ + '.' +
sa_utils.paginate_query.__name__ + '() instead, '
'but note that the arguments differ')
def paginate_query(query, model, limit, sorts, marker_obj=None):
"""Returns a query with sorting / pagination criteria added.
Pagination works by requiring a unique sort key, specified by sorts.
(If sort keys is not unique, then we risk looping through values.)
We use the last row in the previous page as the 'marker' for pagination.
So we must return values that follow the passed marker in the order.
With a single-valued sort key, this would be easy: sort_key > X.
With a compound-values sort key, (k1, k2, k3) we must do this to repeat
the lexicographical ordering:
(k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
The reason of didn't use OFFSET clause was it don't scale, please refer
discussion at https://lists.launchpad.net/openstack/msg02547.html
We also have to cope with different sort directions.
Typically, the id of the last row is used as the client-facing pagination
marker, then the actual marker object must be fetched from the db and
passed in to us as marker.
:param query: the query object to which we should add paging/sorting
:param model: the ORM model class
:param limit: maximum number of items to return
:param sorts: array of attributes and direction by which results should
be sorted
:param marker: the last item of the previous page; we returns the next
results after this value.
:rtype: sqlalchemy.orm.query.Query
:return: The query with sorting/pagination added.
"""
if not sorts:
return query
# A primary key must be specified in sort keys
assert not (limit and
len(set(dict(sorts).keys()) &
set(model.__table__.primary_key.columns.keys())) == 0)
# Add sorting
for sort_key, sort_direction in sorts:
sort_dir_func = sqlalchemy.asc if sort_direction else sqlalchemy.desc
try:
sort_key_attr = getattr(model, sort_key)
except AttributeError:
# Extension attribute doesn't support for sorting. Because it
# existed in attr_info, it will be caught here
msg = _("%s is invalid attribute for sort_key") % sort_key
raise n_exc.BadRequest(resource=model.__tablename__, msg=msg)
if isinstance(sort_key_attr.property, properties.RelationshipProperty):
msg = _("The attribute '%(attr)s' is reference to other "
"resource, can't used by sort "
"'%(resource)s'") % {'attr': sort_key,
'resource': model.__tablename__}
raise n_exc.BadRequest(resource=model.__tablename__, msg=msg)
query = query.order_by(sort_dir_func(sort_key_attr))
# Add pagination
if marker_obj:
marker_values = [getattr(marker_obj, sort[0]) for sort in sorts]
# Build up an array of sort criteria as in the docstring
criteria_list = []
for i, sort in enumerate(sorts):
crit_attrs = [(getattr(model, sorts[j][0]) == marker_values[j])
for j in moves.range(i)]
model_attr = getattr(model, sort[0])
if sort[1]:
crit_attrs.append((model_attr > marker_values[i]))
else:
crit_attrs.append((model_attr < marker_values[i]))
criteria = sqlalchemy.sql.and_(*crit_attrs)
criteria_list.append(criteria)
f = sqlalchemy.sql.or_(*criteria_list)
query = query.filter(f)
if limit:
query = query.limit(limit)
return query
sort_keys = db_utils.get_and_validate_sort_keys(sorts, model)
sort_dirs = ['asc' if s[1] else 'desc' for s in sorts]
return sa_utils.paginate_query(query, model, limit, marker=marker_obj,
sort_keys=sort_keys, sort_dirs=sort_dirs)