openstack/glance
Code Issues Proposed changes

Implement pagination and sorting in v2

Browse Source 
implements blueprint: api-v2-images-sorting
implements blueprint: api-v2-images-pagination

Change-Id: Ibf38847e45534f25dba31c7ffe516e2e3a29450d
This commit is contained in:
Nikhil Komawar 2012-06-08 16:33:41 -04:00
parent 060a75b506
commit 7e84007c6e
9 changed files with 464 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
glance/api/v2/images.py
View File

@ -22,9 +22,13 @@ from glance.common import exception
from glance.common import utils
from glance.common import wsgi
import glance.db
from glance.openstack.common import cfg
from glance.openstack.common import timeutils
CONF = cfg.CONF
class ImagesController(object):
def __init__(self, db_api=None):
self.db_api = db_api or glance.db.get_api()
@ -75,11 +79,24 @@ class ImagesController(object):
return self._normalize_properties(dict(image))
def index(self, req):
def index(self, req, marker=None, limit=None, sort_key='created_at',
sort_dir='desc'):
#NOTE(bcwaldon): is_public=True gets public images and those
# owned by the authenticated tenant
filters = {'deleted': False, 'is_public': True}
images = self.db_api.image_get_all(req.context, filters=filters)
if limit is None:
limit = CONF.limit_param_default
limit = min(CONF.api_limit_max, limit)
try:
images = self.db_api.image_get_all(req.context, filters=filters,
marker=marker, limit=limit,
sort_key=sort_key,
sort_dir=sort_dir)
except exception.InvalidSortKey as e:
raise webob.exc.HTTPBadRequest(explanation=unicode(e))
except exception.NotFound as e:
raise webob.exc.HTTPBadRequest(explanation=unicode(e))
images = [self._normalize_properties(dict(image)) for image in images]
return [self._append_tags(req.context, image) for image in images]
@ -156,6 +173,43 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
def update(self, request):
return self._parse_image(request)
def _validate_limit(self, limit):
try:
limit = int(limit)
except ValueError:
msg = _("limit param must be an integer")
raise webob.exc.HTTPBadRequest(explanation=msg)
if limit < 0:
msg = _("limit param must be positive")
raise webob.exc.HTTPBadRequest(explanation=msg)
return limit
def _validate_sort_dir(self, sort_dir):
if sort_dir not in ['asc', 'desc']:
msg = _('Invalid sort direction: %s' % sort_dir)
raise webob.exc.HTTPBadRequest(explanation=msg)
return sort_dir
def index(self, request):
limit = request.params.get('limit', None)
marker = request.params.get('marker', None)
sort_dir = request.params.get('sort_dir', 'desc')
query_params = {
'sort_key': request.params.get('sort_key', 'created_at'),
'sort_dir': self._validate_sort_dir(sort_dir),
}
if marker is not None:
query_params['marker'] = marker
if limit is not None:
query_params['limit'] = self._validate_limit(limit)
return query_params
class ResponseSerializer(wsgi.JSONResponseSerializer):
def __init__(self, schema_api):

12
glance/common/config.py
View File

@ -37,10 +37,16 @@ paste_deploy_opts = [
]
common_opts = [
cfg.BoolOpt('allow_additional_image_properties', default=True,
help='Whether to allow users to specify image properties '
'beyond what the image schema provides'),
help=_('Whether to allow users to specify image properties '
'beyond what the image schema provides')),
cfg.StrOpt('data_api', default='glance.db.sqlalchemy.api',
help='Python module path of data access API'),
help=_('Python module path of data access API')),
cfg.IntOpt('limit_param_default', default=25,
help=_('Default value for the number of items returned by a '
'request if not specified explicitly in the request')),
cfg.IntOpt('api_limit_max', default=1000,
help=_('Maximum permissible number of items that could be '
'returned by a request')),
]
CONF = cfg.CONF

4
glance/common/exception.py
View File

@ -117,6 +117,10 @@ class Invalid(GlanceException):
message = _("Data supplied was not valid.")
class InvalidSortKey(Invalid):
message = _("Sort key supplied was not valid.")
class AuthorizationRedirect(GlanceException):
message = _("Redirecting to %(uri)s for authorization.")

27
glance/db/simple/api.py
View File

@ -79,8 +79,33 @@ def image_get(context, image_id, session=None):
return image
def image_get_all(context, filters=None):
def image_get_all(context, filters=None, marker=None, limit=None,
sort_key='created_at', sort_dir='desc'):
reverse = False
start = 0
end = -1
images = DATA['images'].values()
if images and not images[0].get(sort_key):
raise exception.InvalidSortKey()
keyfn = lambda x: x[sort_key]
reverse = sort_dir == 'desc'
images.sort(key=keyfn, reverse=reverse)
if marker is None:
start = 0
else:
for i, image in enumerate(images):
if image['id'] == marker:
start = i + 1
break
else:
raise exception.NotFound
if limit is None:
end = -1
else:
end = start + limit
images = images[start:end]
LOG.info('Listing images: %s' % (images))
return images

5
glance/db/sqlalchemy/api.py
View File

@ -392,7 +392,10 @@ def paginate_query(query, model, limit, sort_keys, marker=None,
'desc': desc,
}[current_sort_dir]
sort_key_attr = getattr(model, current_sort_key)
try:
sort_key_attr = getattr(model, current_sort_key)
except AttributeError:
raise exception.InvalidSortKey()
query = query.order_by(sort_dir_func(sort_key_attr))
# Add pagination

6
glance/registry/api/v1/images.py
View File

@ -33,13 +33,7 @@ from glance.openstack.common import timeutils
logger = logging.getLogger('glance.registry.api.v1.images')
images_opts = [
cfg.IntOpt('limit_param_default', default=25),
cfg.IntOpt('api_limit_max', default=1000),
]
CONF = cfg.CONF
CONF.register_opts(images_opts)
DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size',
'disk_format', 'container_format',

177
glance/tests/functional/v2/test_images.py
View File

@ -478,3 +478,180 @@ class TestImages(functional.FunctionalTest):
self.assertEqual(404, response.status_code)
self.stop_servers()
def test_get_images_with_marker_and_limit(self):
image_ids = []
# Image list should be empty
path = self._url('/images')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(0, len(images))
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-1', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-2', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-3', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Image list should contain 3 images
path = self._url('/images')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(3, len(images))
image_ids = [image['id'] for image in images]
# Image list should only contain last 2 images
# and not the first image which is the marker image
path = self._url('/images?marker=%s' % image_ids[0])
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(2, len(images))
self.assertEqual(images[0]['id'], image_ids[1])
self.assertEqual(images[1]['id'], image_ids[2])
# Ensure bad request for using a invalid marker
path = self._url('/images?marker=foo')
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)
#Set limit as 2
# Image list should only contain first 2 images
path = self._url('/images?limit=2')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(2, len(images))
self.assertEqual(images[0]['id'], image_ids[0])
self.assertEqual(images[1]['id'], image_ids[1])
# Ensure bad request for using a invalid limit
path = self._url('/images?limit=foo')
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)
# Ensure bad request for using a zero limit
path = self._url('/images?limit=0')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(0, len(images))
# Ensure bad request for using a negative limit
path = self._url('/images?limit=-1')
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)
# using limit and marker only second image should be returned
path = self._url('/images?limit=1&marker=%s' % image_ids[0])
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(1, len(images))
self.assertEqual(images[0]['id'], image_ids[1])
path = self._url('/images?limit=25')
response = requests.get(path, headers=self._headers())
images = json.loads(response.text)['images']
self.assertEqual(3, len(images))
# Delete first image
path = self._url('/images/%s' % image_ids[0])
response = requests.delete(path, headers=self._headers())
self.assertEqual(204, response.status_code)
# Ensure bad request for using a deleted image as marker
path = self._url('/images?marker=%s' % image_ids[0])
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)
def test_get_images_with_sorting(self):
image_ids = []
# Image list should be empty
path = self._url('/images')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(0, len(images))
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-1', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-2', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Create an image
path = self._url('/images')
headers = self._headers({'content-type': 'application/json'})
data = json.dumps({'name': 'image-3', 'type': 'kernel'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(200, response.status_code)
image_id = json.loads(response.text)['image']['id']
# Image list should contain 3 images
path = self._url('/images')
response = requests.get(path, headers=self._headers())
self.assertEqual(200, response.status_code)
images = json.loads(response.text)['images']
self.assertEqual(3, len(images))
image_ids = [image['id'] for image in images]
# Sort images using name as sort key and desc as sort dir
path = self._url('/images?sort_key=name&sort_dir=desc')
response = requests.get(path, headers=self._headers())
images = json.loads(response.text)['images']
self.assertEqual(3, len(images))
self.assertEqual(images[0]['name'], 'image-3')
self.assertEqual(images[1]['name'], 'image-2')
self.assertEqual(images[2]['name'], 'image-1')
# Sort images using name as sort key and desc as sort asc
path = self._url('/images?sort_key=name&sort_dir=asc')
response = requests.get(path, headers=self._headers())
images = json.loads(response.text)['images']
self.assertEqual(3, len(images))
self.assertEqual(images[0]['name'], 'image-1')
self.assertEqual(images[1]['name'], 'image-2')
self.assertEqual(images[2]['name'], 'image-3')
# Ensure bad request for using a invalid sort_key
path = self._url('/images?sort_key=foo')
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)
# Ensure bad request for using a invalid sort_dir
path = self._url('/images?sort_dir=foo')
response = requests.get(path, headers=self._headers())
self.assertEqual(400, response.status_code)

4
glance/tests/unit/test_db.py
View File

@ -161,6 +161,10 @@ class TestRegistryDb(BaseDBTestCase):
filters=filters)
self.assertEquals(len(images), 0)
def test_image_get_all_invalid_sort_key(self):
self.assertRaises(exception.InvalidSortKey, db_api.image_get_all,
self.context, sort_key='blah')
class TestDBImageTags(BaseDBTestCase):

196
glance/tests/unit/v2/test_images_resource.py
View File

@ -21,6 +21,7 @@ import webob
import glance.api.v2.images
from glance.common import exception
from glance.common import utils
from glance.openstack.common import cfg
import glance.schema
import glance.tests.unit.utils as unit_test_utils
import glance.tests.utils as test_utils
@ -30,19 +31,116 @@ DATETIME = datetime.datetime(2012, 5, 16, 15, 27, 36, 325355)
ISOTIME = '2012-05-16T15:27:36Z'
CONF = cfg.CONF
UUID1 = 'c80a1a6c-bd1f-41c5-90ee-81afedb1d58d'
UUID2 = 'a85abd86-55b3-4d5b-b0b4-5d0a6e6042fc'
UUID3 = '971ec09a-8067-4bc8-a91f-ae3557f1c4c7'
UUID4 = '6bbe7cc2-eae7-4c0f-b50d-a7160b0c6a86'
TENANT1 = '6838eb7b-6ded-434a-882c-b344c77fe8df'
TENANT2 = '2c014f32-55eb-467d-8fcb-4bd706012f81'
TENANT3 = '5a3e60e8-cfa9-4a9e-a90a-62b42cea92b8'
TENANT4 = 'c6c87f25-8a94-47ed-8c83-053c25f42df4'
class TestImagesController(test_utils.BaseTestCase):
def setUp(self):
super(TestImagesController, self).setUp()
self.db = unit_test_utils.FakeDB()
self._create_images()
self.controller = glance.api.v2.images.ImagesController(self.db)
def _create_images(self):
self.db.reset()
self.images = [
{'id': UUID1, 'owner': TENANT1, 'location': UUID1, 'name': '1'},
{'id': UUID2, 'owner': TENANT1, 'name': '2'},
{'id': UUID3, 'owner': TENANT3, 'name': '3'},
{'id': UUID4, 'owner': TENANT4, 'name': '4'},
]
[self.db.image_create(None, image) for image in self.images]
self.db.image_tag_set_all(None, UUID1, ['ping', 'pong'])
def test_index(self):
self.config(limit_param_default=1, api_limit_max=3)
request = unit_test_utils.get_fake_request()
output = self.controller.index(request)
self.assertEqual(2, len(output))
self.assertEqual(output[0]['id'], unit_test_utils.UUID1)
self.assertEqual(output[1]['id'], unit_test_utils.UUID2)
self.assertEqual(1, len(output))
actual = set([image['id'] for image in output])
expected = set([UUID4])
self.assertEqual(actual, expected)
def test_index_with_marker(self):
self.config(limit_param_default=1, api_limit_max=3)
path = '/images'
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request, marker=UUID3)
actual = set([image['id'] for image in output])
self.assertEquals(1, len(actual))
self.assertTrue(UUID2 in actual)
def test_index_with_limit(self):
path = '/images'
limit = 2
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request, limit=limit)
actual = set([image['id'] for image in output])
self.assertEquals(limit, len(actual))
self.assertTrue(UUID4 in actual)
self.assertTrue(UUID3 in actual)
def test_index_greater_than_limit_max(self):
self.config(limit_param_default=1, api_limit_max=3)
path = '/images'
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request, limit=4)
actual = set([image['id'] for image in output])
self.assertEquals(3, len(actual))
def test_index_default_limit(self):
self.config(limit_param_default=1, api_limit_max=3)
path = '/images'
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request)
actual = set([image['id'] for image in output])
self.assertEquals(1, len(actual))
def test_index_with_sort_dir(self):
path = '/images'
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request, sort_dir='asc', limit=3)
actual = [image['id'] for image in output]
self.assertEquals(3, len(actual))
self.assertEquals(UUID1, actual[0])
self.assertEquals(UUID2, actual[1])
self.assertEquals(UUID3, actual[2])
def test_index_with_sort_key(self):
path = '/images'
request = unit_test_utils.get_fake_request(path)
output = self.controller.index(request, sort_key='id', limit=3)
actual = [image['id'] for image in output]
self.assertEquals(3, len(actual))
self.assertEquals(UUID1, actual[0])
self.assertEquals(UUID2, actual[1])
self.assertEquals(UUID3, actual[2])
def test_index_with_marker_not_found(self):
fake_uuid = utils.generate_uuid()
path = '/images'
request = unit_test_utils.get_fake_request(path)
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.index, request, marker=fake_uuid)
def test_index_invalid_sort_key(self):
path = '/images'
request = unit_test_utils.get_fake_request(path)
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.index, request, sort_key='foo')
def test_index_zero_images(self):
self.db.reset()
@ -52,13 +150,13 @@ class TestImagesController(test_utils.BaseTestCase):
def test_show(self):
request = unit_test_utils.get_fake_request()
output = self.controller.show(request, image_id=unit_test_utils.UUID2)
output = self.controller.show(request, image_id=UUID2)
for key in ['created_at', 'updated_at']:
output.pop(key)
expected = {
'id': unit_test_utils.UUID2,
'name': 'image-name',
'owner': unit_test_utils.TENANT1,
'id': UUID2,
'name': '2',
'owner': TENANT1,
'location': None,
'status': 'queued',
'is_public': False,
@ -82,7 +180,7 @@ class TestImagesController(test_utils.BaseTestCase):
output.pop(key)
expected = {
'name': 'image-1',
'owner': unit_test_utils.TENANT1,
'owner': TENANT1,
'location': None,
'status': 'queued',
'is_public': False,
@ -105,7 +203,7 @@ class TestImagesController(test_utils.BaseTestCase):
output.pop(key)
expected = {
'name': 'image-1',
'owner': unit_test_utils.TENANT1,
'owner': TENANT1,
'location': None,
'status': 'queued',
'is_public': True,
@ -117,13 +215,13 @@ class TestImagesController(test_utils.BaseTestCase):
def test_update(self):
request = unit_test_utils.get_fake_request()
image = {'name': 'image-2'}
output = self.controller.update(request, unit_test_utils.UUID1, image)
output = self.controller.update(request, UUID1, image)
for key in ['id', 'created_at', 'updated_at']:
output.pop(key)
expected = {
'name': 'image-2',
'owner': unit_test_utils.TENANT1,
'location': unit_test_utils.UUID1,
'owner': TENANT1,
'location': UUID1,
'status': 'queued',
'is_public': False,
'tags': ['ping', 'pong'],
@ -137,6 +235,12 @@ class TestImagesController(test_utils.BaseTestCase):
self.assertRaises(webob.exc.HTTPNotFound, self.controller.update,
request, utils.generate_uuid(), image)
def test_index_with_invalid_marker(self):
fake_uuid = utils.generate_uuid()
request = unit_test_utils.get_fake_request()
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.index, request, marker=fake_uuid)
class TestImagesDeserializer(test_utils.BaseTestCase):
@ -211,6 +315,74 @@ class TestImagesDeserializer(test_utils.BaseTestCase):
expected = {'image': {'properties': {}}}
self.assertEqual(expected, output)
def test_index(self):
marker = utils.generate_uuid()
path = '/images?limit=1&marker=%s' % marker
request = unit_test_utils.get_fake_request(path)
expected = {'limit': 1,
'marker': marker,
'sort_key': 'created_at',
'sort_dir': 'desc'}
output = self.deserializer.index(request)
self.assertEqual(output, expected)
def test_index_non_integer_limit(self):
request = unit_test_utils.get_fake_request('/images?limit=blah')
self.assertRaises(webob.exc.HTTPBadRequest,
self.deserializer.index, request)
def test_index_zero_limit(self):
request = unit_test_utils.get_fake_request('/images?limit=0')
expected = {'limit': 0,
'sort_key': 'created_at',
'sort_dir': 'desc'}
output = self.deserializer.index(request)
self.assertEqual(expected, output)
def test_index_negative_limit(self):
request = unit_test_utils.get_fake_request('/images?limit=-1')
self.assertRaises(webob.exc.HTTPBadRequest,
self.deserializer.index, request)
def test_index_fraction(self):
request = unit_test_utils.get_fake_request('/images?limit=1.1')
self.assertRaises(webob.exc.HTTPBadRequest,
self.deserializer.index, request)
def test_index_marker(self):
marker = utils.generate_uuid()
path = '/images?marker=%s' % marker
request = unit_test_utils.get_fake_request(path)
output = self.deserializer.index(request)
self.assertEqual(output.get('marker'), marker)
def test_index_marker_not_specified(self):
request = unit_test_utils.get_fake_request('/images')
output = self.deserializer.index(request)
self.assertFalse('marker' in output)
def test_index_limit_not_specified(self):
request = unit_test_utils.get_fake_request('/images')
output = self.deserializer.index(request)
self.assertFalse('limit' in output)
def test_index_sort_key_id(self):
request = unit_test_utils.get_fake_request('/images?sort_key=id')
output = self.deserializer.index(request)
expected = {'sort_key': 'id', 'sort_dir': 'desc'}
self.assertEqual(output, expected)
def test_index_sort_dir_asc(self):
request = unit_test_utils.get_fake_request('/images?sort_dir=asc')
output = self.deserializer.index(request)
expected = {'sort_key': 'created_at', 'sort_dir': 'asc'}
self.assertEqual(output, expected)
def test_index_sort_dir_bad_value(self):
request = unit_test_utils.get_fake_request('/images?sort_dir=blah')
self.assertRaises(webob.exc.HTTPBadRequest,
self.deserializer.index, request)
class TestImagesDeserializerWithExtendedSchema(test_utils.BaseTestCase):