From 23a4049477e69fa279179a00a31a79d88e5d4ed3 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 20 Jun 2011 22:56:51 -0400 Subject: [PATCH 01/15] adding support to registry server and client for sort_key and sort_dir params --- glance/registry/client.py | 42 ++-- glance/registry/db/api.py | 3 +- glance/registry/server.py | 65 ++++-- tests/stubs.py | 24 +- tests/unit/test_api.py | 433 +++++++++++++++++++++++++++++++++++++ tests/unit/test_clients.py | 351 +++++++++++++++++++++++++++++- 6 files changed, 866 insertions(+), 52 deletions(-) diff --git a/glance/registry/client.py b/glance/registry/client.py index 4ee071cfcc..6fc55afa72 100644 --- a/glance/registry/client.py +++ b/glance/registry/client.py @@ -44,46 +44,52 @@ class RegistryClient(BaseClient): port = port or self.DEFAULT_PORT super(RegistryClient, self).__init__(host, port, use_ssl) - def get_images(self, filters=None, marker=None, limit=None): + def get_images(self, **kwargs): """ Returns a list of image id/name mappings from Registry :param filters: dict of keys & expected values to filter results :param marker: image id after which to start page :param limit: max number of images to return + :param sort_key: results will be ordered by this image attribute + :param sort_dir: direction in which to to order results (asc, desc) """ - params = filters or {} - - if marker != None: - params['marker'] = marker - - if limit != None: - params['limit'] = limit - + params = kwargs.get('filters', {}) + params.update(self._extract_get_params(kwargs)) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data - def get_images_detailed(self, filters=None, marker=None, limit=None): + def get_images_detailed(self, **kwargs): """ Returns a list of detailed image data mappings from Registry :param filters: dict of keys & expected values to filter results :param marker: image id after which to start page :param limit: max number of images to return + :param sort_key: results will be ordered by this image attribute + :param sort_dir: direction in which to to order results (asc, desc) """ - params = filters or {} - - if marker != None: - params['marker'] = marker - - if limit != None: - params['limit'] = limit - + params = kwargs.get('filters', {}) + params.update(self._extract_get_params(kwargs)) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data + def _extract_get_params(self, params): + """ + Attempts to extract a subset of keys from the input dictionary. + + :param params: dict of values to filter + :retval subset of 'params' dict + """ + SUPPORTED_GET_PARAMS = ['marker', 'limit', 'sort_key', 'sort_dir'] + result = {} + for PARAM in SUPPORTED_GET_PARAMS: + if PARAM in params: + result[PARAM] = params[PARAM] + return result + def get_image(self, image_id): """Returns a mapping of image metadata from Registry""" res = self.do_request("GET", "/images/%s" % image_id) diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index 3e8106e4cf..af7c5c3c3d 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -129,7 +129,8 @@ def image_get(context, image_id, session=None): raise exception.NotFound("No image found with ID %s" % image_id) -def image_get_all_public(context, filters=None, marker=None, limit=None): +def image_get_all_public(context, filters=None, marker=None, limit=None, + sort_key=None, sort_dir=None): """Get all public images that match zero or more filters. :param filters: dict of filter keys and values. If a 'properties' diff --git a/glance/registry/server.py b/glance/registry/server.py index 173390ea44..e54b9cd7ed 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -39,6 +39,11 @@ DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size', SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format', 'size_min', 'size_max'] +SUPPORTED_SORT_KEYS = ['name', 'status', 'container_format', 'disk_format', + 'size', 'id', 'created_at', 'updated_at'] + +SUPPORTED_SORT_DIRS = ['asc', 'desc'] + MAX_ITEM_LIMIT = 25 @@ -69,14 +74,7 @@ class Controller(wsgi.Controller): } """ - params = { - 'filters': self._get_filters(req), - 'limit': self._get_limit(req), - } - - if 'marker' in req.str_params: - params['marker'] = self._get_marker(req) - + params = self._get_query_params(req) images = db_api.image_get_all_public(None, **params) results = [] @@ -99,19 +97,29 @@ class Controller(wsgi.Controller): all image model fields. """ - params = { - 'filters': self._get_filters(req), - 'limit': self._get_limit(req), - } - - if 'marker' in req.str_params: - params['marker'] = self._get_marker(req) - + params = self._get_query_params(req) images = db_api.image_get_all_public(None, **params) image_dicts = [make_image_dict(i) for i in images] return dict(images=image_dicts) + def _get_query_params(self, req): + """Extract necessary query parameters from http request. + + :param req: the Request object coming from the wsgi layer + :retval dictionary of filters to apply to list of images + + """ + params = { + 'filters': self._get_filters(req), + 'limit': self._get_limit(req), + 'sort_key': self._get_sort_key(req), + 'sort_dir': self._get_sort_dir(req), + 'marker': self._get_marker(req), + } + + return params + def _get_filters(self, req): """Return a dictionary of query param filters from the request @@ -148,12 +156,35 @@ class Controller(wsgi.Controller): def _get_marker(self, req): """Parse a marker query param into something usable.""" + marker = req.str_params.get('marker', None) + + if marker is None: + return None + try: - marker = int(req.str_params.get('marker', None)) + marker = int(marker) except ValueError: raise exc.HTTPBadRequest("marker param must be an integer") return marker + def _get_sort_key(self, req): + """Parse a sort key query param from the request object.""" + sort_key = req.str_params.get('sort_key', None) + if sort_key is not None and sort_key not in SUPPORTED_SORT_KEYS: + _keys = '. '.join(SUPPORTED_SORT_KEYS) + msg = "Unsupported sort_key. Acceptable values: %s" % (_keys,) + raise exc.HTTPBadRequest(explanation=msg) + return sort_key + + def _get_sort_dir(self, req): + """Parse a sort direction query param from the request object.""" + sort_dir = req.str_params.get('sort_dir', None) + if sort_dir is not None and sort_dir not in SUPPORTED_SORT_DIRS: + _keys = '. '.join(SUPPORTED_SORT_DIRS) + msg = "Unsupported sort_dir. Acceptable values: %s" % (_keys,) + raise exc.HTTPBadRequest(explanation=msg) + return sort_dir + def show(self, req, id): """Return data about the given image id.""" try: diff --git a/tests/stubs.py b/tests/stubs.py index 635fd2ea07..e5e497a4aa 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -388,8 +388,8 @@ def stub_out_registry_db_image_api(stubs): else: return images[0] - def image_get_all_public(self, _context, filters=None, - marker=None, limit=1000): + def image_get_all_public(self, _context, filters=None, marker=None, + limit=1000, sort_key=None, sort_dir=None): images = [f for f in self.images if f['is_public'] == True] if 'size_min' in filters: @@ -414,16 +414,24 @@ def stub_out_registry_db_image_api(stubs): for k, v in filters.items(): images = [f for f in images if f[k] == v] + # sorted func expects func that compares in descending order def image_cmp(x, y): - if x['created_at'] > y['created_at']: - return 1 - elif x['created_at'] == y['created_at']: + _sort_dir = sort_dir or 'desc' + multiplier = { + 'asc': -1, + 'desc': 1, + }[_sort_dir] + + _sort_key = sort_key or 'created_at' + if x[_sort_key] > y[_sort_key]: + return 1 * multiplier + elif x[_sort_key] == y[_sort_key]: if x['id'] > y['id']: - return 1 + return 1 * multiplier else: - return -1 + return -1 * multiplier else: - return -1 + return -1 * multiplier images = sorted(images, cmp=image_cmp) images.reverse() diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 1055fe98df..82401c8f67 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -285,6 +285,398 @@ class TestRegistryAPI(unittest.TestCase): for image in images: self.assertEqual('new name! #123', image['name']) + def test_get_index_sort_default_created_at_desc(self): + """Tests that the /images registry API returns list of + public images that conforms to a default sort key/dir + + """ + time1 = datetime.datetime.utcnow() + datetime.timedelta(seconds=5) + time2 = datetime.datetime.utcnow() + + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 19, + 'checksum': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 5, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': time2} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images') + res = req.get_response(self.api) + res_dict = json.loads(res.body) + self.assertEquals(res.status_int, 200) + + images = res_dict['images'] + self.assertEquals(len(images), 4) + self.assertEquals(int(images[0]['id']), 4) + self.assertEquals(int(images[1]['id']), 3) + self.assertEquals(int(images[2]['id']), 5) + self.assertEquals(int(images[3]['id']), 2) + + def test_get_index_bad_sort_key(self): + """Ensure a 400 is returned when a bad sort_key is provided.""" + req = webob.Request.blank('/images?sort_key=asdf') + res = req.get_response(self.api) + self.assertEqual(400, res.status_int) + + def test_get_index_bad_sort_dir(self): + """Ensure a 400 is returned when a bad sort_dir is provided.""" + req = webob.Request.blank('/images?sort_dir=asdf') + res = req.get_response(self.api) + self.assertEqual(400, res.status_int) + + def test_get_index_sort_id_desc(self): + """Tests that the /images registry API returns list of + public images sorted by id in descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=id&sort_dir=desc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 4) + self.assertEquals(int(images[1]['id']), 3) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_name_asc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by name in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=name&sort_dir=asc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 4) + + def test_get_index_sort_status_desc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by status in + descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'killed', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=status&sort_dir=desc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_disk_format_asc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by disk_format in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vdi', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=disk_format&sort_dir=asc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_container_format_desc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by container_format in + descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'iso', + 'container_format': 'bare', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + url = '/images?sort_key=container_format&sort_dir=desc' + req = webob.Request.blank(url) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 2) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_size_asc(self): + """Tests that the /images registry API returns list of + public images sorted by size in ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 100, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'iso', + 'container_format': 'bare', + 'name': 'xyz', + 'size': 2, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + url = '/images?sort_key=size&sort_dir=asc' + req = webob.Request.blank(url) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 4) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_created_at_asc(self): + """Tests that the /images registry API returns list of + public images sorted by created_at in ascending order. + + """ + now = datetime.datetime.utcnow() + time1 = now + datetime.timedelta(seconds=5) + time2 = now + + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 19, + 'checksum': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': time2} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=created_at&sort_dir=asc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 2) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_updated_at_desc(self): + """Tests that the /images registry API returns list of + public images sorted by updated_at in descending order. + + """ + now = datetime.datetime.utcnow() + time1 = now + datetime.timedelta(seconds=5) + time2 = now + + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 19, + 'checksum': None, + 'created_at': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': None, + 'updated_at': time2} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=updated_at&sort_dir=desc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + def test_get_details(self): """Tests that the /images/detail registry API returns a mapping containing a list of detailed image information @@ -668,6 +1060,47 @@ class TestRegistryAPI(unittest.TestCase): for image in images: self.assertEqual('v a', image['properties']['prop_123']) + def test_get_details_sort_name_asc(self): + """Tests that the /images/details registry API returns list of + public images sorted alphabetically by name in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images/detail?sort_key=name&sort_dir=asc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 4) + + + def test_create_image(self): """Tests that the /images POST registry API creates the image""" fixture = {'name': 'fake public image', diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 2be1a3ad9e..56e5babdb0 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import json import os import stubout @@ -70,6 +71,299 @@ class TestRegistryClient(unittest.TestCase): for k, v in fixture.items(): self.assertEquals(v, images[0][k]) + def test_get_index_sort_id_desc(self): + """Tests that the /images registry API returns list of + public images sorted by id in descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='id', sort_dir='desc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 4) + self.assertEquals(int(images[1]['id']), 3) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_name_asc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by name in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='name', sort_dir='asc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 4) + + def test_get_index_sort_status_desc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by status in + descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'killed', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='status', sort_dir='desc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_disk_format_asc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by disk_format in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vdi', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='disk_format', + sort_dir='asc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_index_sort_container_format_desc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by container_format in + descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'iso', + 'container_format': 'bare', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='container_format', + sort_dir='desc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 2) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_size_asc(self): + """Tests that the /images registry API returns list of + public images sorted by size in ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 100, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'iso', + 'container_format': 'bare', + 'name': 'xyz', + 'size': 2, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='size', sort_dir='asc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 4) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_created_at_asc(self): + """Tests that the /images registry API returns list of + public images sorted by created_at in ascending order. + + """ + now = datetime.datetime.utcnow() + time1 = now + datetime.timedelta(seconds=5) + time2 = now + + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 19, + 'checksum': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': time2} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='created_at', sort_dir='asc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 2) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 3) + + def test_get_index_sort_updated_at_desc(self): + """Tests that the /images registry API returns list of + public images sorted by updated_at in descending order. + + """ + now = datetime.datetime.utcnow() + time1 = now + datetime.timedelta(seconds=5) + time2 = now + + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 19, + 'checksum': None, + 'created_at': None, + 'created_at': time1} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'new name! #123', + 'size': 20, + 'checksum': None, + 'created_at': None, + 'updated_at': time2} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='updated_at', sort_dir='desc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_image_index_marker(self): """Test correct set of images returned with marker param.""" extra_fixture = {'id': 3, @@ -170,7 +464,7 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images({'name': 'new name! #123'}) + images = self.client.get_images(filters={'name': 'new name! #123'}) self.assertEquals(len(images), 1) for image in images: @@ -236,7 +530,8 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'name': 'new name! #123'}) + filters = {'name': 'new name! #123'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 1) for image in images: @@ -255,7 +550,7 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'status': 'saving'}) + images = self.client.get_images_detailed(filters={'status': 'saving'}) self.assertEquals(len(images), 1) for image in images: @@ -274,7 +569,8 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'container_format': 'ovf'}) + filters = {'container_format': 'ovf'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 2) for image in images: @@ -293,7 +589,8 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'disk_format': 'vhd'}) + filters = {'disk_format': 'vhd'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 2) for image in images: @@ -312,7 +609,7 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'size_max': 20}) + images = self.client.get_images_detailed(filters={'size_max': 20}) self.assertEquals(len(images), 1) for image in images: @@ -331,7 +628,7 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'size_min': 20}) + images = self.client.get_images_detailed(filters={'size_min': 20}) self.assertEquals(len(images), 1) for image in images: @@ -351,12 +648,50 @@ class TestRegistryClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'property-p a': 'v a'}) + filters = {'property-p a': 'v a'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 1) for image in images: self.assertEquals('v a', image['properties']['p a']) + def test_get_image_details_sort_disk_format_asc(self): + """Tests that a detailed call returns list of + public images sorted alphabetically by disk_format in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vdi', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images_detailed(sort_key='disk_format', + sort_dir='asc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 2) + + def test_get_image(self): """Tests that the detailed info about an image returned""" fixture = {'id': 1, From 1d454f7595763e5ae0acd498f3749bf55a72bfbd Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 20 Jun 2011 23:42:03 -0400 Subject: [PATCH 02/15] adding ordering support to glance api --- glance/api/v1/images.py | 29 +++++--- glance/client.py | 36 ++++++---- glance/registry/client.py | 14 ---- glance/registry/db/api.py | 21 ++++-- tests/functional/test_curl_api.py | 115 ++++++++++++++++++++++++++++++ tests/unit/test_api.py | 41 ++++++++++- tests/unit/test_clients.py | 50 +++++++++++-- 7 files changed, 257 insertions(+), 49 deletions(-) diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 0d799f537c..e041f1c03b 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -92,14 +92,7 @@ class Controller(wsgi.Controller): 'size': }, ... ]} """ - params = {'filters': self._get_filters(req)} - - if 'limit' in req.str_params: - params['limit'] = req.str_params.get('limit') - - if 'marker' in req.str_params: - params['marker'] = req.str_params.get('marker') - + params = self._get_query_params(req) images = registry.get_images_list(self.options, **params) return dict(images=images) @@ -125,6 +118,17 @@ class Controller(wsgi.Controller): 'properties': {'distro': 'Ubuntu 10.04 LTS', ...}}, ... ]} """ + params = self._get_query_params(req) + images = registry.get_images_detail(self.options, **params) + return dict(images=images) + + def _get_query_params(self, req): + """ + Extracts necessary query params from request. + + :param req: the WSGI Request object + :retval dict of parameters that can be used by registry client + """ params = {'filters': self._get_filters(req)} if 'limit' in req.str_params: @@ -133,8 +137,13 @@ class Controller(wsgi.Controller): if 'marker' in req.str_params: params['marker'] = req.str_params.get('marker') - images = registry.get_images_detail(self.options, **params) - return dict(images=images) + if 'sort_key' in req.str_params: + params['sort_key'] = req.str_params.get('sort_key') + + if 'sort_dir' in req.str_params: + params['sort_dir'] = req.str_params.get('sort_dir') + + return params def _get_filters(self, req): """ diff --git a/glance/client.py b/glance/client.py index 7a01c7d141..503c1f362e 100644 --- a/glance/client.py +++ b/glance/client.py @@ -183,6 +183,20 @@ class BaseClient(object): else: return response.status + def _extract_get_params(self, params): + """ + Attempts to extract a subset of keys from the input dictionary. + + :param params: dict of values to filter + :retval subset of 'params' dict + """ + SUPPORTED_GET_PARAMS = ['marker', 'limit', 'sort_key', 'sort_dir'] + result = {} + for PARAM in SUPPORTED_GET_PARAMS: + if PARAM in params: + result[PARAM] = params[PARAM] + return result + class V1Client(BaseClient): @@ -209,7 +223,7 @@ class V1Client(BaseClient): return super(V1Client, self).do_request(method, action, body, headers, params) - def get_images(self, filters=None, marker=None, limit=None): + def get_images(self, **kwargs): """ Returns a list of image id/name mappings from Registry @@ -217,18 +231,17 @@ class V1Client(BaseClient): collection of images should be filtered :param marker: id after which to start the page of images :param limit: maximum number of items to return + :param sort_key: results will be ordered by this image attribute + :param sort_dir: direction in which to to order results (asc, desc) """ - params = filters or {} - if marker: - params['marker'] = marker - if limit: - params['limit'] = limit + params = kwargs.get('filters', {}) + params.update(self._extract_get_params(kwargs)) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data - def get_images_detailed(self, filters=None, marker=None, limit=None): + def get_images_detailed(self, **kwargs): """ Returns a list of detailed image data mappings from Registry @@ -236,13 +249,12 @@ class V1Client(BaseClient): collection of images should be filtered :param marker: id after which to start the page of images :param limit: maximum number of items to return + :param sort_key: results will be ordered by this image attribute + :param sort_dir: direction in which to to order results (asc, desc) """ - params = filters or {} - if marker: - params['marker'] = marker - if limit: - params['limit'] = limit + params = kwargs.get('filters', {}) + params.update(self._extract_get_params(kwargs)) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data diff --git a/glance/registry/client.py b/glance/registry/client.py index 6fc55afa72..15e69c1b6e 100644 --- a/glance/registry/client.py +++ b/glance/registry/client.py @@ -76,20 +76,6 @@ class RegistryClient(BaseClient): data = json.loads(res.read())['images'] return data - def _extract_get_params(self, params): - """ - Attempts to extract a subset of keys from the input dictionary. - - :param params: dict of values to filter - :retval subset of 'params' dict - """ - SUPPORTED_GET_PARAMS = ['marker', 'limit', 'sort_key', 'sort_dir'] - result = {} - for PARAM in SUPPORTED_GET_PARAMS: - if PARAM in params: - result[PARAM] = params[PARAM] - return result - def get_image(self, image_id): """Returns a mapping of image metadata from Registry""" res = self.do_request("GET", "/images/%s" % image_id) diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index af7c5c3c3d..72332260bf 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -23,7 +23,7 @@ Defines interface for DB access import logging -from sqlalchemy import create_engine, desc +from sqlalchemy import asc, create_engine, desc from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import exc from sqlalchemy.orm import joinedload @@ -138,7 +138,8 @@ def image_get_all_public(context, filters=None, marker=None, limit=None, filters on the image properties attribute :param marker: image id after which to start page :param limit: maximum number of images to return - + :param sort_key: image attribute by which results should be sorted + :param sort_dir: direction in which results should be sorted (asc, desc) """ filters = filters or {} @@ -147,9 +148,19 @@ def image_get_all_public(context, filters=None, marker=None, limit=None, options(joinedload(models.Image.properties)).\ filter_by(deleted=_deleted(context)).\ filter_by(is_public=True).\ - filter(models.Image.status != 'killed').\ - order_by(desc(models.Image.created_at)).\ - order_by(desc(models.Image.id)) + filter(models.Image.status != 'killed') + + _sort_dir = sort_dir or 'desc' + sort_dir_func = { + 'asc': asc, + 'desc': desc, + }[_sort_dir] + + _sort_key = sort_key or 'created_at' + sort_key_attr = getattr(models.Image, _sort_key) + + query = query.order_by(sort_dir_func(sort_key_attr)).\ + order_by(sort_dir_func(models.Image.id)) if 'size_min' in filters: query = query.filter(models.Image.size >= filters['size_min']) diff --git a/tests/functional/test_curl_api.py b/tests/functional/test_curl_api.py index c16e6b0bea..b4e25f7a65 100644 --- a/tests/functional/test_curl_api.py +++ b/tests/functional/test_curl_api.py @@ -1117,3 +1117,118 @@ class TestCurlApi(functional.FunctionalTest): self.assertEqual(int(images[0]['id']), 2) self.stop_servers() + + def test_ordered_images(self): + """ + Set up three test images and ensure each query param filter works + """ + self.cleanup() + self.start_servers() + + api_port = self.api_port + registry_port = self.registry_port + + # 0. GET /images + # Verify no public images + cmd = "curl http://0.0.0.0:%d/v1/images" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + self.assertEqual('{"images": []}', out.strip()) + + # 1. POST /images with three public images with various attributes + cmd = ("curl -i -X POST " + "-H 'Expect: ' " # Necessary otherwise sends 100 Continue + "-H 'X-Image-Meta-Name: Image1' " + "-H 'X-Image-Meta-Status: active' " + "-H 'X-Image-Meta-Container-Format: ovf' " + "-H 'X-Image-Meta-Disk-Format: vdi' " + "-H 'X-Image-Meta-Size: 19' " + "-H 'X-Image-Meta-Is-Public: True' " + "http://0.0.0.0:%d/v1/images") % api_port + + exitcode, out, err = execute(cmd) + self.assertEqual(0, exitcode) + + lines = out.split("\r\n") + status_line = lines[0] + + self.assertEqual("HTTP/1.1 201 Created", status_line) + + cmd = ("curl -i -X POST " + "-H 'Expect: ' " # Necessary otherwise sends 100 Continue + "-H 'X-Image-Meta-Name: ASDF' " + "-H 'X-Image-Meta-Status: active' " + "-H 'X-Image-Meta-Container-Format: bare' " + "-H 'X-Image-Meta-Disk-Format: iso' " + "-H 'X-Image-Meta-Size: 2' " + "-H 'X-Image-Meta-Is-Public: True' " + "http://0.0.0.0:%d/v1/images") % api_port + + exitcode, out, err = execute(cmd) + self.assertEqual(0, exitcode) + + lines = out.split("\r\n") + status_line = lines[0] + + self.assertEqual("HTTP/1.1 201 Created", status_line) + cmd = ("curl -i -X POST " + "-H 'Expect: ' " # Necessary otherwise sends 100 Continue + "-H 'X-Image-Meta-Name: XYZ' " + "-H 'X-Image-Meta-Status: saving' " + "-H 'X-Image-Meta-Container-Format: ami' " + "-H 'X-Image-Meta-Disk-Format: ami' " + "-H 'X-Image-Meta-Size: 5' " + "-H 'X-Image-Meta-Is-Public: True' " + "http://0.0.0.0:%d/v1/images") % api_port + + exitcode, out, err = execute(cmd) + self.assertEqual(0, exitcode) + + lines = out.split("\r\n") + status_line = lines[0] + + self.assertEqual("HTTP/1.1 201 Created", status_line) + + # 2. GET /images with no query params + # Verify three public images sorted by created_at desc + cmd = "curl http://0.0.0.0:%d/v1/images" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + images = json.loads(out.strip())['images'] + + self.assertEqual(len(images), 3) + self.assertEqual(images[0]['id'], 3) + self.assertEqual(images[1]['id'], 2) + self.assertEqual(images[2]['id'], 1) + + # 3. GET /images sorted by name asc + params = 'sort_key=name&sort_dir=asc' + cmd = "curl 'http://0.0.0.0:%d/v1/images?%s'" % (api_port, params) + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + images = json.loads(out.strip())['images'] + + self.assertEqual(len(images), 3) + self.assertEqual(images[0]['id'], 2) + self.assertEqual(images[1]['id'], 1) + self.assertEqual(images[2]['id'], 3) + + # 4. GET /images sorted by size desc + params = 'sort_key=size&sort_dir=desc' + cmd = "curl 'http://0.0.0.0:%d/v1/images?%s'" % (api_port, params) + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + images = json.loads(out.strip())['images'] + + self.assertEqual(len(images), 3) + self.assertEqual(images[0]['id'], 1) + self.assertEqual(images[1]['id'], 3) + self.assertEqual(images[2]['id'], 2) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 82401c8f67..bd62156dad 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1099,8 +1099,6 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[1]['id']), 2) self.assertEquals(int(images[2]['id']), 4) - - def test_create_image(self): """Tests that the /images POST registry API creates the image""" fixture = {'name': 'fake public image', @@ -1449,6 +1447,45 @@ class TestGlanceAPI(unittest.TestCase): "res.headerlist = %r" % res.headerlist) self.assertTrue('/images/3' in res.headers['location']) + def test_get_index_sort_name_asc(self): + """Tests that the /images registry API returns list of + public images sorted alphabetically by name in + ascending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images?sort_key=name&sort_dir=asc') + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 3) + self.assertEquals(int(images[1]['id']), 2) + self.assertEquals(int(images[2]['id']), 4) + def test_image_is_checksummed(self): """Test that the image contents are checksummed properly""" fixture_headers = {'x-image-meta-store': 'file', diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 56e5babdb0..2f78dd1e4d 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -363,7 +363,6 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[1]['id']), 4) self.assertEquals(int(images[2]['id']), 2) - def test_get_image_index_marker(self): """Test correct set of images returned with marker param.""" extra_fixture = {'id': 3, @@ -691,7 +690,6 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[1]['id']), 4) self.assertEquals(int(images[2]['id']), 2) - def test_get_image(self): """Tests that the detailed info about an image returned""" fixture = {'id': 1, @@ -897,6 +895,42 @@ class TestClient(unittest.TestCase): self.client.get_image, 3) + def test_get_image_index_sort_container_format_desc(self): + """Tests that the client returns list of public images + sorted alphabetically by container_format in + descending order. + + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': True, + 'disk_format': 'ami', + 'container_format': 'ami', + 'name': 'asdf', + 'size': 19, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + extra_fixture = {'id': 4, + 'status': 'active', + 'is_public': True, + 'disk_format': 'iso', + 'container_format': 'bare', + 'name': 'xyz', + 'size': 20, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + images = self.client.get_images(sort_key='container_format', + sort_dir='desc') + + self.assertEquals(len(images), 3) + self.assertEquals(int(images[0]['id']), 2) + self.assertEquals(int(images[1]['id']), 4) + self.assertEquals(int(images[2]['id']), 3) + def test_get_image_index(self): """Test correct set of public image returned""" fixture = {'id': 2, @@ -1006,7 +1040,8 @@ class TestClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images({'name': 'new name! #123'}) + filters = {'name': 'new name! #123'} + images = self.client.get_images(filters=filters) self.assertEquals(len(images), 1) self.assertEquals('new name! #123', images[0]['name']) @@ -1025,7 +1060,8 @@ class TestClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images({'property-p a': 'v a'}) + filters = {'property-p a': 'v a'} + images = self.client.get_images(filters=filters) self.assertEquals(len(images), 1) self.assertEquals(3, images[0]['id']) @@ -1100,7 +1136,8 @@ class TestClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'name': 'new name! #123'}) + filters = {'name': 'new name! #123'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 1) for image in images: @@ -1120,7 +1157,8 @@ class TestClient(unittest.TestCase): glance.registry.db.api.image_create(None, extra_fixture) - images = self.client.get_images_detailed({'property-p a': 'v a'}) + filters = {'property-p a': 'v a'} + images = self.client.get_images_detailed(filters=filters) self.assertEquals(len(images), 1) for image in images: From 3ebfff8ba89a389ce3cb16c969601c1f0fd8851d Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 20 Jun 2011 23:52:25 -0400 Subject: [PATCH 03/15] updating docs --- doc/source/glanceapi.rst | 14 ++++++++++++++ doc/source/registries.rst | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/doc/source/glanceapi.rst b/doc/source/glanceapi.rst index b978e9832d..beec85bd79 100644 --- a/doc/source/glanceapi.rst +++ b/doc/source/glanceapi.rst @@ -129,6 +129,20 @@ list details these query parameters. Filters images having a ``size`` attribute less than or equal to ``BYTES`` +These two resources also accept sort parameters: + +* ``sort_key=KEY`` + + Results will be ordered by the specified image attribute ``KEY``. Accepted + values include ``id``, ``name``, ``status``, ``disk_format``, + ``container_format``, ``size``, ``created_at`` (default) and ``updated_at``. + +* ``sort_dir=DIR`` + + Results will be sorted in the direction ``DIR``. Accepted values are ``asc`` + for ascending or ``desc`` (default) for descending. + + Requesting Detailed Metadata on a Specific Image ------------------------------------------------ diff --git a/doc/source/registries.rst b/doc/source/registries.rst index a949bbe253..b59ad41c68 100644 --- a/doc/source/registries.rst +++ b/doc/source/registries.rst @@ -83,6 +83,20 @@ list details these query parameters. Filters images having a ``size`` attribute less than or equal to ``BYTES`` +These two resources also accept sort parameters: + +* ``sort_key=KEY`` + + Results will be ordered by the specified image attribute ``KEY``. Accepted + values include ``id``, ``name``, ``status``, ``disk_format``, + ``container_format``, ``size``, ``created_at`` (default) and ``updated_at``. + +* ``sort_dir=DIR`` + + Results will be sorted in the direction ``DIR``. Accepted values are ``asc`` + for ascending or ``desc`` (default) for descending. + + ``POST /images`` ---------------- From a21bcc9e558d77fd53c81e050ae5389a301cee9f Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 20 Jun 2011 23:58:24 -0400 Subject: [PATCH 04/15] slight refactoring --- glance/api/v1/images.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index e041f1c03b..a7950ac7b5 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -130,19 +130,10 @@ class Controller(wsgi.Controller): :retval dict of parameters that can be used by registry client """ params = {'filters': self._get_filters(req)} - - if 'limit' in req.str_params: - params['limit'] = req.str_params.get('limit') - - if 'marker' in req.str_params: - params['marker'] = req.str_params.get('marker') - - if 'sort_key' in req.str_params: - params['sort_key'] = req.str_params.get('sort_key') - - if 'sort_dir' in req.str_params: - params['sort_dir'] = req.str_params.get('sort_dir') - + SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') + for PARAM in SUPPORTED_PARAMS: + if PARAM in req.str_params: + params[PARAM] = req.str_params.get(PARAM) return params def _get_filters(self, req): From 9ea2bd11ab501cdc3822ccd81ee0f8910c3438ca Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 21 Jun 2011 00:01:39 -0400 Subject: [PATCH 05/15] making SUPPORTED_* lists into tuples --- glance/client.py | 2 +- glance/registry/server.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glance/client.py b/glance/client.py index 503c1f362e..5b98b7c40f 100644 --- a/glance/client.py +++ b/glance/client.py @@ -190,7 +190,7 @@ class BaseClient(object): :param params: dict of values to filter :retval subset of 'params' dict """ - SUPPORTED_GET_PARAMS = ['marker', 'limit', 'sort_key', 'sort_dir'] + SUPPORTED_GET_PARAMS = ('marker', 'limit', 'sort_key', 'sort_dir') result = {} for PARAM in SUPPORTED_GET_PARAMS: if PARAM in params: diff --git a/glance/registry/server.py b/glance/registry/server.py index e54b9cd7ed..318153389d 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -39,10 +39,10 @@ DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size', SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format', 'size_min', 'size_max'] -SUPPORTED_SORT_KEYS = ['name', 'status', 'container_format', 'disk_format', - 'size', 'id', 'created_at', 'updated_at'] +SUPPORTED_SORT_KEYS = ('name', 'status', 'container_format', 'disk_format', + 'size', 'id', 'created_at', 'updated_at') -SUPPORTED_SORT_DIRS = ['asc', 'desc'] +SUPPORTED_SORT_DIRS = ('asc', 'desc') MAX_ITEM_LIMIT = 25 From cb127f68e553306f4b92096cdff4022255eb0e43 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 21 Jun 2011 00:11:00 -0400 Subject: [PATCH 06/15] fixing bad request error messages --- glance/registry/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glance/registry/server.py b/glance/registry/server.py index 318153389d..15d002baba 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -171,7 +171,7 @@ class Controller(wsgi.Controller): """Parse a sort key query param from the request object.""" sort_key = req.str_params.get('sort_key', None) if sort_key is not None and sort_key not in SUPPORTED_SORT_KEYS: - _keys = '. '.join(SUPPORTED_SORT_KEYS) + _keys = ', '.join(SUPPORTED_SORT_KEYS) msg = "Unsupported sort_key. Acceptable values: %s" % (_keys,) raise exc.HTTPBadRequest(explanation=msg) return sort_key @@ -180,7 +180,7 @@ class Controller(wsgi.Controller): """Parse a sort direction query param from the request object.""" sort_dir = req.str_params.get('sort_dir', None) if sort_dir is not None and sort_dir not in SUPPORTED_SORT_DIRS: - _keys = '. '.join(SUPPORTED_SORT_DIRS) + _keys = ', '.join(SUPPORTED_SORT_DIRS) msg = "Unsupported sort_dir. Acceptable values: %s" % (_keys,) raise exc.HTTPBadRequest(explanation=msg) return sort_dir From d55c2a3d97203262ed62cec2ce07492994a01e69 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 21 Jun 2011 09:41:06 -0400 Subject: [PATCH 07/15] updating client docs --- doc/source/client.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/doc/source/client.rst b/doc/source/client.rst index 247840bf02..bbda9a0987 100644 --- a/doc/source/client.rst +++ b/doc/source/client.rst @@ -113,7 +113,36 @@ in size and in the `saving` status. c = Client("glance.example.com", 9292) filters = {'status': 'saving', 'size_max': (5 * 1024 * 1024 * 1024)} - print c.get_images_detailed(filters) + print c.get_images_detailed(filters=filters) + +Sorting Images Returned via ``get_images()`` and ``get_images_detailed()`` +-------------------------------------------------------------------------- + +Two parameters are available to sort the list of images returned by +these methods. + +* ``sort_key: KEY`` + + Images can be ordered by the image attribute ``KEY``. Acceptable values: + ``id``, ``name``, ``status``, ``container_format``, ``disk_format``, + ``created_at`` (default) and ``updated_at``. + +* ``sort_dir: DIR`` + + The direction of the sort may be defined by ``DIR``. Accepted values: + ``asc`` for ascending or ``desc`` (default) for descending. + +The following example will return a list of images sorted alphabetically +by name in ascending order. + +.. code-block:: python + + from glance.client import Client + + c = Client("glance.example.com", 9292) + + print c.get_images(sort_key='name', sort_dir='asc') + Requesting Detailed Metadata on a Specific Image ------------------------------------------------ From 220be9cd83b202222f4137f792d9ac270dbc963b Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 21 Jun 2011 16:27:52 -0400 Subject: [PATCH 08/15] refactoring for Jay --- glance/api/v1/images.py | 3 ++- glance/client.py | 13 +++++++------ glance/registry/client.py | 6 ++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index a7950ac7b5..8579bfc4e7 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -45,6 +45,8 @@ logger = logging.getLogger('glance.api.v1.images') SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format', 'size_min', 'size_max'] +SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') + class Controller(wsgi.Controller): @@ -130,7 +132,6 @@ class Controller(wsgi.Controller): :retval dict of parameters that can be used by registry client """ params = {'filters': self._get_filters(req)} - SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') for PARAM in SUPPORTED_PARAMS: if PARAM in req.str_params: params[PARAM] = req.str_params.get(PARAM) diff --git a/glance/client.py b/glance/client.py index 5b98b7c40f..c0c63011f7 100644 --- a/glance/client.py +++ b/glance/client.py @@ -33,6 +33,10 @@ from glance.common import exception #TODO(jaypipes) Allow a logger param for client classes +# parameters accepted in get_images* methods +SUPPORTED_GET_PARAMS = ('marker', 'limit', 'sort_key', 'sort_dir') + + class ClientConnectionError(Exception): """Error resulting from a client connecting to a server""" pass @@ -190,8 +194,7 @@ class BaseClient(object): :param params: dict of values to filter :retval subset of 'params' dict """ - SUPPORTED_GET_PARAMS = ('marker', 'limit', 'sort_key', 'sort_dir') - result = {} + result = params.get('filters', {}) for PARAM in SUPPORTED_GET_PARAMS: if PARAM in params: result[PARAM] = params[PARAM] @@ -235,8 +238,7 @@ class V1Client(BaseClient): :param sort_dir: direction in which to to order results (asc, desc) """ - params = kwargs.get('filters', {}) - params.update(self._extract_get_params(kwargs)) + params = self._extract_get_params(kwargs) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data @@ -253,8 +255,7 @@ class V1Client(BaseClient): :param sort_dir: direction in which to to order results (asc, desc) """ - params = kwargs.get('filters', {}) - params.update(self._extract_get_params(kwargs)) + params = self._extract_get_params(kwargs) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data diff --git a/glance/registry/client.py b/glance/registry/client.py index 15e69c1b6e..090081d4de 100644 --- a/glance/registry/client.py +++ b/glance/registry/client.py @@ -54,8 +54,7 @@ class RegistryClient(BaseClient): :param sort_key: results will be ordered by this image attribute :param sort_dir: direction in which to to order results (asc, desc) """ - params = kwargs.get('filters', {}) - params.update(self._extract_get_params(kwargs)) + params = self._extract_get_params(kwargs) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data @@ -70,8 +69,7 @@ class RegistryClient(BaseClient): :param sort_key: results will be ordered by this image attribute :param sort_dir: direction in which to to order results (asc, desc) """ - params = kwargs.get('filters', {}) - params.update(self._extract_get_params(kwargs)) + params = self._extract_get_params(kwargs) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data From a2bd04212f5038e57495c858fb001f71ddf8ab2f Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Sun, 26 Jun 2011 16:43:41 -0400 Subject: [PATCH 09/15] restructuring client code --- bin/glance | 2 +- glance/api/v1/__init__.py | 2 +- glance/client.py | 187 ++----------------------------------- glance/common/exception.py | 5 + glance/registry/client.py | 7 +- glance/registry/server.py | 2 + tests/stubs.py | 5 +- tests/unit/test_clients.py | 2 +- 8 files changed, 24 insertions(+), 188 deletions(-) diff --git a/bin/glance b/bin/glance index 77db2fe876..b99d2d713e 100755 --- a/bin/glance +++ b/bin/glance @@ -194,7 +194,7 @@ EXAMPLES print "Returned the following metadata for the new image:" for k, v in sorted(image_meta.items()): print " %(k)30s => %(v)s" % locals() - except client.ClientConnectionError, e: + except exception.ClientConnectionError, e: host = options.host port = options.port print ("Failed to connect to the Glance API server " diff --git a/glance/api/v1/__init__.py b/glance/api/v1/__init__.py index 3daeaab09f..3a9e97ff9d 100644 --- a/glance/api/v1/__init__.py +++ b/glance/api/v1/__init__.py @@ -19,7 +19,6 @@ import logging import routes -from glance.api.v1 import images from glance.common import wsgi logger = logging.getLogger('glance.api.v1') @@ -30,6 +29,7 @@ class API(wsgi.Router): """WSGI router for Glance v1 API requests.""" def __init__(self, options): + from glance.api.v1 import images self.options = options mapper = routes.Mapper() resource = images.create_resource(options) diff --git a/glance/client.py b/glance/client.py index c0c63011f7..df150f0365 100644 --- a/glance/client.py +++ b/glance/client.py @@ -19,189 +19,17 @@ Client classes for callers of a Glance system """ -import httplib import json -import logging -import urlparse -import socket -import sys -import urllib -from glance import utils +from glance.api.v1 import images as v1_images +from glance.common import client as base_client from glance.common import exception +from glance import utils #TODO(jaypipes) Allow a logger param for client classes -# parameters accepted in get_images* methods -SUPPORTED_GET_PARAMS = ('marker', 'limit', 'sort_key', 'sort_dir') - - -class ClientConnectionError(Exception): - """Error resulting from a client connecting to a server""" - pass - - -class ImageBodyIterator(object): - - """ - A class that acts as an iterator over an image file's - chunks of data. This is returned as part of the result - tuple from `glance.client.Client.get_image` - """ - - CHUNKSIZE = 65536 - - def __init__(self, response): - """ - Constructs the object from an HTTPResponse object - """ - self.response = response - - def __iter__(self): - """ - Exposes an iterator over the chunks of data in the - image file. - """ - while True: - chunk = self.response.read(ImageBodyIterator.CHUNKSIZE) - if chunk: - yield chunk - else: - break - - -class BaseClient(object): - - """A base client class""" - - CHUNKSIZE = 65536 - - def __init__(self, host, port, use_ssl): - """ - Creates a new client to some service. - - :param host: The host where service resides - :param port: The port where service resides - :param use_ssl: Should we use HTTPS? - """ - self.host = host - self.port = port - self.use_ssl = use_ssl - self.connection = None - - def get_connection_type(self): - """ - Returns the proper connection type - """ - if self.use_ssl: - return httplib.HTTPSConnection - else: - return httplib.HTTPConnection - - def do_request(self, method, action, body=None, headers=None, - params=None): - """ - Connects to the server and issues a request. Handles converting - any returned HTTP error status codes to OpenStack/Glance exceptions - and closing the server connection. Returns the result data, or - raises an appropriate exception. - - :param method: HTTP method ("GET", "POST", "PUT", etc...) - :param action: part of URL after root netloc - :param body: string of data to send, or None (default) - :param headers: mapping of key/value pairs to add as headers - :param params: dictionary of key/value pairs to add to append - to action - - :note - - If the body param has a read attribute, and method is either - POST or PUT, this method will automatically conduct a chunked-transfer - encoding and use the body as a file object, transferring chunks - of data using the connection's send() method. This allows large - objects to be transferred efficiently without buffering the entire - body in memory. - """ - if type(params) is dict: - action += '?' + urllib.urlencode(params) - - try: - connection_type = self.get_connection_type() - headers = headers or {} - c = connection_type(self.host, self.port) - - # Do a simple request or a chunked request, depending - # on whether the body param is a file-like object and - # the method is PUT or POST - if hasattr(body, 'read') and method.lower() in ('post', 'put'): - # Chunk it, baby... - c.putrequest(method, action) - - for header, value in headers.items(): - c.putheader(header, value) - c.putheader('Transfer-Encoding', 'chunked') - c.endheaders() - - chunk = body.read(self.CHUNKSIZE) - while chunk: - c.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = body.read(self.CHUNKSIZE) - c.send('0\r\n\r\n') - else: - # Simple request... - c.request(method, action, body, headers) - res = c.getresponse() - status_code = self.get_status_code(res) - if status_code in (httplib.OK, - httplib.CREATED, - httplib.ACCEPTED, - httplib.NO_CONTENT): - return res - elif status_code == httplib.UNAUTHORIZED: - raise exception.NotAuthorized - elif status_code == httplib.FORBIDDEN: - raise exception.NotAuthorized - elif status_code == httplib.NOT_FOUND: - raise exception.NotFound - elif status_code == httplib.CONFLICT: - raise exception.Duplicate(res.read()) - elif status_code == httplib.BAD_REQUEST: - raise exception.Invalid(res.read()) - elif status_code == httplib.INTERNAL_SERVER_ERROR: - raise Exception("Internal Server error: %s" % res.read()) - else: - raise Exception("Unknown error occurred! %s" % res.read()) - - except (socket.error, IOError), e: - raise ClientConnectionError("Unable to connect to " - "server. Got error: %s" % e) - - def get_status_code(self, response): - """ - Returns the integer status code from the response, which - can be either a Webob.Response (used in testing) or httplib.Response - """ - if hasattr(response, 'status_int'): - return response.status_int - else: - return response.status - - def _extract_get_params(self, params): - """ - Attempts to extract a subset of keys from the input dictionary. - - :param params: dict of values to filter - :retval subset of 'params' dict - """ - result = params.get('filters', {}) - for PARAM in SUPPORTED_GET_PARAMS: - if PARAM in params: - result[PARAM] = params[PARAM] - return result - - -class V1Client(BaseClient): +class V1Client(base_client.BaseClient): """Main client class for accessing Glance resources""" @@ -237,8 +65,7 @@ class V1Client(BaseClient): :param sort_key: results will be ordered by this image attribute :param sort_dir: direction in which to to order results (asc, desc) """ - - params = self._extract_get_params(kwargs) + params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data @@ -255,7 +82,7 @@ class V1Client(BaseClient): :param sort_dir: direction in which to to order results (asc, desc) """ - params = self._extract_get_params(kwargs) + params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data @@ -273,7 +100,7 @@ class V1Client(BaseClient): res = self.do_request("GET", "/images/%s" % image_id) image = utils.get_image_meta_from_headers(res) - return image, ImageBodyIterator(res) + return image, base_client.ImageBodyIterator(res) def get_image_meta(self, image_id): """ diff --git a/glance/common/exception.py b/glance/common/exception.py index e2ad116f2f..351c3e276a 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -83,6 +83,11 @@ class DatabaseMigrationError(Error): pass +class ClientConnectionError(Exception): + """Error resulting from a client connecting to a server""" + pass + + def wrap_exception(f): def _wrap(*args, **kw): try: diff --git a/glance/registry/client.py b/glance/registry/client.py index a5a2302008..ff84f8b143 100644 --- a/glance/registry/client.py +++ b/glance/registry/client.py @@ -23,7 +23,8 @@ the Glance Registry API import json import urllib -from glance.client import BaseClient +from glance.common.client import BaseClient +from glance.registry import server class RegistryClient(BaseClient): @@ -54,7 +55,7 @@ class RegistryClient(BaseClient): :param sort_key: results will be ordered by this image attribute :param sort_dir: direction in which to to order results (asc, desc) """ - params = self._extract_get_params(kwargs) + params = self._extract_params(kwargs, server.SUPPORTED_PARAMS) res = self.do_request("GET", "/images", params=params) data = json.loads(res.read())['images'] return data @@ -69,7 +70,7 @@ class RegistryClient(BaseClient): :param sort_key: results will be ordered by this image attribute :param sort_dir: direction in which to to order results (asc, desc) """ - params = self._extract_get_params(kwargs) + params = self._extract_params(kwargs, server.SUPPORTED_PARAMS) res = self.do_request("GET", "/images/detail", params=params) data = json.loads(res.read())['images'] return data diff --git a/glance/registry/server.py b/glance/registry/server.py index 0154de93c3..bedb63cc26 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -46,6 +46,8 @@ SUPPORTED_SORT_DIRS = ('asc', 'desc') MAX_ITEM_LIMIT = 25 +SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') + class Controller(object): """Controller for the reference implementation registry server""" diff --git a/tests/stubs.py b/tests/stubs.py index e5e497a4aa..f281a2473b 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -28,6 +28,7 @@ import sys import stubout import webob +import glance.common.client from glance.common import exception from glance.registry import server as rserver from glance.api import v1 as server @@ -254,9 +255,9 @@ def stub_out_registry_and_store_server(stubs): for i in self.response.app_iter: yield i - stubs.Set(glance.client.BaseClient, 'get_connection_type', + stubs.Set(glance.common.client.BaseClient, 'get_connection_type', fake_get_connection_type) - stubs.Set(glance.client.ImageBodyIterator, '__iter__', + stubs.Set(glance.common.client.ImageBodyIterator, '__iter__', fake_image_iter) diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 2f78dd1e4d..663c304b23 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -38,7 +38,7 @@ class TestBadClients(unittest.TestCase): def test_bad_address(self): """Test ClientConnectionError raised""" c = client.Client("127.999.1.1") - self.assertRaises(client.ClientConnectionError, + self.assertRaises(exception.ClientConnectionError, c.get_image, 1) From c1ff5c38bfcc757ccff69276f8374503aa6d745c Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Sun, 26 Jun 2011 16:45:46 -0400 Subject: [PATCH 10/15] adding base client module --- glance/common/client.py | 169 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 glance/common/client.py diff --git a/glance/common/client.py b/glance/common/client.py new file mode 100644 index 0000000000..f8ed40efd5 --- /dev/null +++ b/glance/common/client.py @@ -0,0 +1,169 @@ +import httplib +import logging +import socket +import urllib + +from glance.common import exception + + +class ImageBodyIterator(object): + + """ + A class that acts as an iterator over an image file's + chunks of data. This is returned as part of the result + tuple from `glance.client.Client.get_image` + """ + + CHUNKSIZE = 65536 + + def __init__(self, response): + """ + Constructs the object from an HTTPResponse object + """ + self.response = response + + def __iter__(self): + """ + Exposes an iterator over the chunks of data in the + image file. + """ + while True: + chunk = self.response.read(ImageBodyIterator.CHUNKSIZE) + if chunk: + yield chunk + else: + break + + +class BaseClient(object): + + """A base client class""" + + CHUNKSIZE = 65536 + + def __init__(self, host, port, use_ssl): + """ + Creates a new client to some service. + + :param host: The host where service resides + :param port: The port where service resides + :param use_ssl: Should we use HTTPS? + """ + self.host = host + self.port = port + self.use_ssl = use_ssl + self.connection = None + + def get_connection_type(self): + """ + Returns the proper connection type + """ + if self.use_ssl: + return httplib.HTTPSConnection + else: + return httplib.HTTPConnection + + def do_request(self, method, action, body=None, headers=None, + params=None): + """ + Connects to the server and issues a request. Handles converting + any returned HTTP error status codes to OpenStack/Glance exceptions + and closing the server connection. Returns the result data, or + raises an appropriate exception. + + :param method: HTTP method ("GET", "POST", "PUT", etc...) + :param action: part of URL after root netloc + :param body: string of data to send, or None (default) + :param headers: mapping of key/value pairs to add as headers + :param params: dictionary of key/value pairs to add to append + to action + + :note + + If the body param has a read attribute, and method is either + POST or PUT, this method will automatically conduct a chunked-transfer + encoding and use the body as a file object, transferring chunks + of data using the connection's send() method. This allows large + objects to be transferred efficiently without buffering the entire + body in memory. + """ + if type(params) is dict: + action += '?' + urllib.urlencode(params) + + try: + connection_type = self.get_connection_type() + headers = headers or {} + c = connection_type(self.host, self.port) + + # Do a simple request or a chunked request, depending + # on whether the body param is a file-like object and + # the method is PUT or POST + if hasattr(body, 'read') and method.lower() in ('post', 'put'): + # Chunk it, baby... + c.putrequest(method, action) + + for header, value in headers.items(): + c.putheader(header, value) + c.putheader('Transfer-Encoding', 'chunked') + c.endheaders() + + chunk = body.read(self.CHUNKSIZE) + while chunk: + c.send('%x\r\n%s\r\n' % (len(chunk), chunk)) + chunk = body.read(self.CHUNKSIZE) + c.send('0\r\n\r\n') + else: + # Simple request... + c.request(method, action, body, headers) + res = c.getresponse() + status_code = self.get_status_code(res) + if status_code in (httplib.OK, + httplib.CREATED, + httplib.ACCEPTED, + httplib.NO_CONTENT): + return res + elif status_code == httplib.UNAUTHORIZED: + raise exception.NotAuthorized + elif status_code == httplib.FORBIDDEN: + raise exception.NotAuthorized + elif status_code == httplib.NOT_FOUND: + raise exception.NotFound + elif status_code == httplib.CONFLICT: + raise exception.Duplicate(res.read()) + elif status_code == httplib.BAD_REQUEST: + raise exception.Invalid(res.read()) + elif status_code == httplib.INTERNAL_SERVER_ERROR: + raise Exception("Internal Server error: %s" % res.read()) + else: + raise Exception("Unknown error occurred! %s" % res.read()) + + except (socket.error, IOError), e: + raise exception.ClientConnectionError("Unable to connect to " + "server. Got error: %s" % e) + + def get_status_code(self, response): + """ + Returns the integer status code from the response, which + can be either a Webob.Response (used in testing) or httplib.Response + """ + if hasattr(response, 'status_int'): + return response.status_int + else: + return response.status + + def _extract_params(self, actual_params, allowed_params): + """ + Extract a subset of keys from a dictionary. The filters key + will also be extracted, and each of its values will be returned + as an individual param. + + :param actual_params: dict of keys to filter + :param allowed_params: list of keys that 'actual_params' will be + reduced to + :retval subset of 'params' dict + """ + result = actual_params.get('filters', {}) + for allowed_param in allowed_params: + if allowed_param in actual_params: + result[allowed_param] = actual_params[allowed_param] + return result From 763dbc4d6ed607f9362af3f070b80071d755a922 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Sun, 26 Jun 2011 16:56:17 -0400 Subject: [PATCH 11/15] cleaning up None values being passed into images_get_all_public db call --- glance/registry/db/api.py | 8 +++----- glance/registry/server.py | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index 72332260bf..cae03a0111 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -130,7 +130,7 @@ def image_get(context, image_id, session=None): def image_get_all_public(context, filters=None, marker=None, limit=None, - sort_key=None, sort_dir=None): + sort_key='created_at', sort_dir='desc'): """Get all public images that match zero or more filters. :param filters: dict of filter keys and values. If a 'properties' @@ -150,14 +150,12 @@ def image_get_all_public(context, filters=None, marker=None, limit=None, filter_by(is_public=True).\ filter(models.Image.status != 'killed') - _sort_dir = sort_dir or 'desc' sort_dir_func = { 'asc': asc, 'desc': desc, - }[_sort_dir] + }[sort_dir] - _sort_key = sort_key or 'created_at' - sort_key_attr = getattr(models.Image, _sort_key) + sort_key_attr = getattr(models.Image, sort_key) query = query.order_by(sort_dir_func(sort_key_attr)).\ order_by(sort_dir_func(models.Image.id)) diff --git a/glance/registry/server.py b/glance/registry/server.py index bedb63cc26..a992cdabc3 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -120,6 +120,10 @@ class Controller(object): 'marker': self._get_marker(req), } + for key, value in params.items(): + if value is None: + del params[key] + return params def _get_filters(self, req): From 6214a48000534f680f911e5b0860c3830387ecf7 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 27 Jun 2011 14:00:09 -0400 Subject: [PATCH 12/15] docstring --- glance/registry/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/glance/registry/server.py b/glance/registry/server.py index a992cdabc3..622f74eb50 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -110,7 +110,6 @@ class Controller(object): :param req: the Request object coming from the wsgi layer :retval dictionary of filters to apply to list of images - """ params = { 'filters': self._get_filters(req), From c8b40ee7a7a89e431b49174a7dbd83a0921238f2 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 27 Jun 2011 14:05:29 -0400 Subject: [PATCH 13/15] reverting one import change; another docstring fix --- glance/api/v1/__init__.py | 2 +- glance/registry/server.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/glance/api/v1/__init__.py b/glance/api/v1/__init__.py index 3a9e97ff9d..3daeaab09f 100644 --- a/glance/api/v1/__init__.py +++ b/glance/api/v1/__init__.py @@ -19,6 +19,7 @@ import logging import routes +from glance.api.v1 import images from glance.common import wsgi logger = logging.getLogger('glance.api.v1') @@ -29,7 +30,6 @@ class API(wsgi.Router): """WSGI router for Glance v1 API requests.""" def __init__(self, options): - from glance.api.v1 import images self.options = options mapper = routes.Mapper() resource = images.create_resource(options) diff --git a/glance/registry/server.py b/glance/registry/server.py index 622f74eb50..012268140f 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -106,7 +106,8 @@ class Controller(object): return dict(images=image_dicts) def _get_query_params(self, req): - """Extract necessary query parameters from http request. + """ + Extract necessary query parameters from http request. :param req: the Request object coming from the wsgi layer :retval dictionary of filters to apply to list of images From c7b4168c3cfb6806938aeac650bc2d9009a0de33 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 28 Jun 2011 09:41:50 -0400 Subject: [PATCH 14/15] docstrings\! --- tests/unit/test_api.py | 43 +++++++++++++++++++------------------- tests/unit/test_clients.py | 40 +++++++++++++++++------------------ 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index c7e63d6f96..12f299980e 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -286,9 +286,9 @@ class TestRegistryAPI(unittest.TestCase): self.assertEqual('new name! #123', image['name']) def test_get_index_sort_default_created_at_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images that conforms to a default sort key/dir - """ time1 = datetime.datetime.utcnow() + datetime.timedelta(seconds=5) time2 = datetime.datetime.utcnow() @@ -354,9 +354,9 @@ class TestRegistryAPI(unittest.TestCase): self.assertEqual(400, res.status_int) def test_get_index_sort_id_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by id in descending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -392,10 +392,10 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_name_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by name in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -431,10 +431,10 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 4) def test_get_index_sort_status_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by status in descending order. - """ extra_fixture = {'id': 3, 'status': 'killed', @@ -470,10 +470,10 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_disk_format_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by disk_format in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -509,10 +509,10 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_container_format_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by container_format in descending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -549,9 +549,9 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_size_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by size in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -588,9 +588,9 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_created_at_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by created_at in ascending order. - """ now = datetime.datetime.utcnow() time1 = now + datetime.timedelta(seconds=5) @@ -632,9 +632,9 @@ class TestRegistryAPI(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_updated_at_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by updated_at in descending order. - """ now = datetime.datetime.utcnow() time1 = now + datetime.timedelta(seconds=5) @@ -1061,10 +1061,10 @@ class TestRegistryAPI(unittest.TestCase): self.assertEqual('v a', image['properties']['prop_123']) def test_get_details_sort_name_asc(self): - """Tests that the /images/details registry API returns list of + """ + Tests that the /images/details registry API returns list of public images sorted alphabetically by name in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -1474,7 +1474,8 @@ class TestGlanceAPI(unittest.TestCase): self.assertTrue('/images/3' in res.headers['location']) def test_get_index_sort_name_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by name in ascending order. diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 663c304b23..58be704ee2 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -72,9 +72,9 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(v, images[0][k]) def test_get_index_sort_id_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by id in descending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -106,10 +106,10 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_name_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by name in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -141,10 +141,10 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 4) def test_get_index_sort_status_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by status in descending order. - """ extra_fixture = {'id': 3, 'status': 'killed', @@ -176,10 +176,10 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_disk_format_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by disk_format in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -212,10 +212,10 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 2) def test_get_index_sort_container_format_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted alphabetically by container_format in descending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -248,9 +248,9 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_size_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by size in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -282,9 +282,9 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_created_at_asc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by created_at in ascending order. - """ now = datetime.datetime.utcnow() time1 = now + datetime.timedelta(seconds=5) @@ -322,9 +322,9 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals(int(images[2]['id']), 3) def test_get_index_sort_updated_at_desc(self): - """Tests that the /images registry API returns list of + """ + Tests that the /images registry API returns list of public images sorted by updated_at in descending order. - """ now = datetime.datetime.utcnow() time1 = now + datetime.timedelta(seconds=5) @@ -655,10 +655,10 @@ class TestRegistryClient(unittest.TestCase): self.assertEquals('v a', image['properties']['p a']) def test_get_image_details_sort_disk_format_asc(self): - """Tests that a detailed call returns list of + """ + Tests that a detailed call returns list of public images sorted alphabetically by disk_format in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active', @@ -896,10 +896,10 @@ class TestClient(unittest.TestCase): 3) def test_get_image_index_sort_container_format_desc(self): - """Tests that the client returns list of public images + """ + Tests that the client returns list of public images sorted alphabetically by container_format in descending order. - """ extra_fixture = {'id': 3, 'status': 'active', From 8c5da9d57a8e7e3b53446b975fe6f2c6f81d5812 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 28 Jun 2011 09:51:08 -0400 Subject: [PATCH 15/15] fixing one last docstring --- tests/unit/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 12f299980e..43f31b29ce 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1478,7 +1478,6 @@ class TestGlanceAPI(unittest.TestCase): Tests that the /images registry API returns list of public images sorted alphabetically by name in ascending order. - """ extra_fixture = {'id': 3, 'status': 'active',