Implement paginate query use marker in nova-api
1.add limit and marker param to db.instance_get_all_by_filters() 2.set the filter use marker 3.execute limit before sqlarchmy get_all() 4.add testcase 'test_db_api.test_instance_get_all_by_filters_paginate' 5.related testcase: test_get_servers_with_marker() test_get_servers_with_limit_and_marker() in nova/tests/api/openstack/compute/test_servers.py test_instance_get_all_by_filters_paginate() in nova/tests/test_db_api.py 6.add InvalidSortkey exception Implement bp:efficient-limiting. Change-Id: Iea3eeb7b51194b6017d624506aafc6469d7338e4
This commit is contained in:
parent
07a5196ccb
commit
84710606d5
128
nova/common/sqlalchemyutils.py
Normal file
128
nova/common/sqlalchemyutils.py
Normal file
@ -0,0 +1,128 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2010-2011 OpenStack LLC.
|
||||
# Copyright 2012 Justin Santa Barbara
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Implementation of paginate query."""
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from nova import exception
|
||||
from nova.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# copy from glance/db/sqlalchemy/api.py
|
||||
def paginate_query(query, model, limit, sort_keys, marker=None,
|
||||
sort_dir=None, sort_dirs=None):
|
||||
"""Returns a query with sorting / pagination criteria added.
|
||||
|
||||
Pagination works by requiring a unique sort_key, specified by sort_keys.
|
||||
(If sort_keys is not unique, then we risk looping through values.)
|
||||
We use the last row in the previous page as the 'marker' for pagination.
|
||||
So we must return values that follow the passed marker in the order.
|
||||
With a single-valued sort_key, this would be easy: sort_key > X.
|
||||
With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
|
||||
the lexicographical ordering:
|
||||
(k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
|
||||
|
||||
We also have to cope with different sort_directions.
|
||||
|
||||
Typically, the id of the last row is used as the client-facing pagination
|
||||
marker, then the actual marker object must be fetched from the db and
|
||||
passed in to us as marker.
|
||||
|
||||
:param query: the query object to which we should add paging/sorting
|
||||
:param model: the ORM model class
|
||||
:param limit: maximum number of items to return
|
||||
:param sort_keys: array of attributes by which results should be sorted
|
||||
:param marker: the last item of the previous page; we returns the next
|
||||
results after this value.
|
||||
:param sort_dir: direction in which results should be sorted (asc, desc)
|
||||
:param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys
|
||||
|
||||
:rtype: sqlalchemy.orm.query.Query
|
||||
:return: The query with sorting/pagination added.
|
||||
"""
|
||||
|
||||
if 'id' not in sort_keys:
|
||||
# TODO(justinsb): If this ever gives a false-positive, check
|
||||
# the actual primary key, rather than assuming its id
|
||||
LOG.warn(_('Id not in sort_keys; is sort_keys unique?'))
|
||||
|
||||
assert(not (sort_dir and sort_dirs))
|
||||
|
||||
# Default the sort direction to ascending
|
||||
if sort_dirs is None and sort_dir is None:
|
||||
sort_dir = 'asc'
|
||||
|
||||
# Ensure a per-column sort direction
|
||||
if sort_dirs is None:
|
||||
sort_dirs = [sort_dir for _sort_key in sort_keys]
|
||||
|
||||
assert(len(sort_dirs) == len(sort_keys))
|
||||
|
||||
# Add sorting
|
||||
for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs):
|
||||
sort_dir_func = {
|
||||
'asc': sqlalchemy.asc,
|
||||
'desc': sqlalchemy.desc,
|
||||
}[current_sort_dir]
|
||||
|
||||
try:
|
||||
sort_key_attr = getattr(model, current_sort_key)
|
||||
except AttributeError:
|
||||
raise exception.InvalidSortKey()
|
||||
query = query.order_by(sort_dir_func(sort_key_attr))
|
||||
|
||||
# Add pagination
|
||||
if marker is not None:
|
||||
marker_values = []
|
||||
for sort_key in sort_keys:
|
||||
v = getattr(marker, sort_key)
|
||||
marker_values.append(v)
|
||||
|
||||
# Build up an array of sort criteria as in the docstring
|
||||
criteria_list = []
|
||||
for i in xrange(0, len(sort_keys)):
|
||||
crit_attrs = []
|
||||
for j in xrange(0, i):
|
||||
model_attr = getattr(model, sort_keys[j])
|
||||
crit_attrs.append((model_attr == marker_values[j]))
|
||||
|
||||
model_attr = getattr(model, sort_keys[i])
|
||||
if sort_dirs[i] == 'desc':
|
||||
crit_attrs.append((model_attr < marker_values[i]))
|
||||
elif sort_dirs[i] == 'asc':
|
||||
crit_attrs.append((model_attr > marker_values[i]))
|
||||
else:
|
||||
raise ValueError(_("Unknown sort direction, "
|
||||
"must be 'desc' or 'asc'"))
|
||||
|
||||
criteria = sqlalchemy.sql.and_(*crit_attrs)
|
||||
criteria_list.append(criteria)
|
||||
|
||||
f = sqlalchemy.sql.or_(*criteria_list)
|
||||
query = query.filter(f)
|
||||
|
||||
if limit is not None:
|
||||
query = query.limit(limit)
|
||||
|
||||
return query
|
@ -294,6 +294,10 @@ class InvalidGroup(Invalid):
|
||||
message = _("Group not valid. Reason: %(reason)s")
|
||||
|
||||
|
||||
class InvalidSortKey(Invalid):
|
||||
message = _("Sort key supplied was not valid.")
|
||||
|
||||
|
||||
class InstanceInvalidState(Invalid):
|
||||
message = _("Instance %(instance_uuid)s in %(attr)s %(state)s. Cannot "
|
||||
"%(method)s while the instance is in this state.")
|
||||
|
@ -113,6 +113,32 @@ class DbApiTestCase(test.TestCase):
|
||||
else:
|
||||
self.assertTrue(result[1].deleted)
|
||||
|
||||
def test_instance_get_all_by_filters_paginate(self):
|
||||
self.flags(sql_connection="notdb://")
|
||||
test1 = self.create_instances_with_args(display_name='test1')
|
||||
test2 = self.create_instances_with_args(display_name='test2')
|
||||
test3 = self.create_instances_with_args(display_name='test3')
|
||||
|
||||
result = db.instance_get_all_by_filters(self.context,
|
||||
{'display_name': '%test%'},
|
||||
marker=None)
|
||||
self.assertEqual(3, len(result))
|
||||
result = db.instance_get_all_by_filters(self.context,
|
||||
{'display_name': '%test%'},
|
||||
sort_dir="asc",
|
||||
marker=test1)
|
||||
self.assertEqual(2, len(result))
|
||||
result = db.instance_get_all_by_filters(self.context,
|
||||
{'display_name': '%test%'},
|
||||
sort_dir="asc",
|
||||
marker=test2)
|
||||
self.assertEqual(1, len(result))
|
||||
result = db.instance_get_all_by_filters(self.context,
|
||||
{'display_name': '%test%'},
|
||||
sort_dir="asc",
|
||||
marker=test3)
|
||||
self.assertEqual(0, len(result))
|
||||
|
||||
def test_migration_get_unconfirmed_by_dest_compute(self):
|
||||
ctxt = context.get_admin_context()
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user