Browse Source

Add limit, offset, order in collection GET

Allow limiting the number of objects returned via GET
by providing "limit"
Example: api/notifications?limit=5

Allow offseting (skipping N first records) via "offset"
Example: api/notifications?offset=100

Allow ordering of objects by providing "order_by"
Example: api/notifications?order_by=-id

Add helper functions/classes to:
- get HTTP parameters (limit, offset, order_by)
- get scoped collection query by applying 4 operations
  filter, order, offset, limit
- set Conent-Range header if scope limits are present

Make default NailgunCollection's GET utilize scoped query
This makes default (parent) GET of child handlers support paging
and ordering (overriden GET methods will not get this functionality
automatically)
NailgunCollection.GET is also an example of how to implement
this new functionality.

Helper functions/classes can be utilized in child handler methods
to implement filters / ordering / paging

Related-Bug: 1657348
Change-Id: I7760465f70b3f69791e7a0c558a26e8ba55c934a
tags/11.0.0.0rc1
Dmitry Sutyagin 2 years ago
parent
commit
03aaca2dee

+ 90
- 1
nailgun/nailgun/api/v1/handlers/base.py View File

@@ -123,6 +123,15 @@ class BaseHandler(object):
123 123
                     data=self.message
124 124
                 )
125 125
 
126
+        class _range_not_satisfiable(web.HTTPError):
127
+            message = 'Requested Range Not Satisfiable'
128
+
129
+            def __init__(self):
130
+                super(_range_not_satisfiable, self).__init__(
131
+                    status='416 Range Not Satisfiable',
132
+                    data=self.message
133
+                )
134
+
126 135
         exc_status_map = {
127 136
             200: web.ok,
128 137
             201: web.created,
@@ -141,6 +150,7 @@ class BaseHandler(object):
141 150
             409: web.conflict,
142 151
             410: web.gone,
143 152
             415: web.unsupportedmediatype,
153
+            416: _range_not_satisfiable,
144 154
 
145 155
             500: web.internalerror,
146 156
         }
@@ -445,12 +455,86 @@ class SingleHandler(BaseHandler):
445 455
         raise self.http(204)
446 456
 
447 457
 
458
+class Pagination(object):
459
+    """Get pagination scope from init or HTTP request arguments"""
460
+
461
+    def convert(self, x):
462
+        """ret. None if x=None, else ret. x as int>=0; else raise 400"""
463
+
464
+        val = x
465
+        if val is not None:
466
+            if type(val) is not int:
467
+                try:
468
+                    val = int(x)
469
+                except ValueError:
470
+                    raise BaseHandler.http(400, 'Cannot convert "%s" to int'
471
+                                           % x)
472
+            # raise on negative values
473
+            if val < 0:
474
+                raise BaseHandler.http(400, 'Negative limit/offset not \
475
+                                       allowed')
476
+            return val
477
+
478
+    def get_order_by(self, order_by):
479
+        if order_by:
480
+            order_by = [s.strip() for s in order_by.split(',') if s.strip()]
481
+        return order_by if order_by else None
482
+
483
+    def __init__(self, limit=None, offset=None, order_by=None):
484
+        if limit is not None or offset is not None or order_by is not None:
485
+            # init with provided arguments
486
+            self.limit = self.convert(limit)
487
+            self.offset = self.convert(offset)
488
+            self.order_by = self.get_order_by(order_by)
489
+        else:
490
+            # init with HTTP arguments
491
+            self.limit = self.convert(web.input(limit=None).limit)
492
+            self.offset = self.convert(web.input(offset=None).offset)
493
+            self.order_by = self.get_order_by(web.input(order_by=None)
494
+                                              .order_by)
495
+
496
+
448 497
 class CollectionHandler(BaseHandler):
449 498
 
450 499
     collection = None
451 500
     validator = BasicValidator
452 501
     eager = ()
453 502
 
503
+    def get_scoped_query_and_range(self, pagination=None, filter_by=None):
504
+        """Get filtered+paged collection query and collection.ContentRange obj
505
+
506
+        Return a scoped query, and if pagination is requested then also return
507
+        ContentRange object (see NailgunCollection.content_range) to allow to
508
+        set Content-Range header (outside of this functon).
509
+        If pagination is not set/requested, return query to all collection's
510
+        objects.
511
+        Allows getting object count without getting objects - via
512
+        content_range if pagination.limit=0.
513
+
514
+        :param pagination: Pagination object
515
+        :param filter_by: filter dict passed to query.filter_by(\*\*dict)
516
+        :type filter_by: dict
517
+        :returns: SQLAlchemy query and ContentRange object
518
+        """
519
+        pagination = pagination or Pagination()
520
+        query = None
521
+        content_range = None
522
+        if self.collection and self.collection.single.model:
523
+            query, content_range = self.collection.scope(pagination, filter_by)
524
+            if content_range:
525
+                if not content_range.valid:
526
+                    raise self.http(416, 'Requested range "%s" cannot be '
527
+                                    'satisfied' % content_range)
528
+        return query, content_range
529
+
530
+    def set_content_range(self, content_range):
531
+        """Set Content-Range header to indicate partial data
532
+
533
+        :param content_range: NailgunCollection.content_range named tuple
534
+        """
535
+        txt = 'objects {x.first}-{x.last}/{x.total}'.format(x=content_range)
536
+        web.header('Content-Range', txt)
537
+
454 538
     @handle_errors
455 539
     @validate
456 540
     @serialize
@@ -458,8 +542,13 @@ class CollectionHandler(BaseHandler):
458 542
         """:returns: Collection of JSONized REST objects.
459 543
 
460 544
         :http: * 200 (OK)
545
+               * 400 (Bad Request)
546
+               * 406 (requested range not satisfiable)
461 547
         """
462
-        q = self.collection.eager(None, self.eager)
548
+        query, content_range = self.get_scoped_query_and_range()
549
+        if content_range:
550
+            self.set_content_range(content_range)
551
+        q = self.collection.eager(query, self.eager)
463 552
         return self.collection.to_list(q)
464 553
 
465 554
     @handle_errors

+ 103
- 24
nailgun/nailgun/objects/base.py View File

@@ -169,6 +169,30 @@ class NailgunCollection(object):
169 169
     #: Single object class
170 170
     single = NailgunObject
171 171
 
172
+    @classmethod
173
+    def content_range(cls, first, last, total, valid):
174
+        """Structure to set Content-Range header
175
+
176
+        Defines structure necessary to implement paged requests.
177
+        "total" is needed to let client calculate how many pages are available.
178
+        "valid" is used to indicate that the requested page is valid
179
+        (contains data) or not (outside of data range).
180
+        Used in NailgunCollection.scope()
181
+
182
+        :param first: first element (row) returned
183
+        :param last: last element (row) returned
184
+        :param total: total number of elements/rows (before pagination)
185
+        :param valid: whether the pagination is within data range or not
186
+        :returns: ContentRange object (collections.namedtuple) with 4 fields
187
+        """
188
+        rng = collections.namedtuple('ContentRange',
189
+                                     ['first', 'last', 'total', 'valid'])
190
+        rng.first = first
191
+        rng.last = last
192
+        rng.total = total
193
+        rng.valid = valid
194
+        return rng
195
+
172 196
     @classmethod
173 197
     def _is_iterable(cls, obj):
174 198
         return isinstance(
@@ -191,6 +215,53 @@ class NailgunCollection(object):
191 215
         """
192 216
         return db().query(cls.single.model)
193 217
 
218
+    @classmethod
219
+    def scope(cls, pagination=None, filter_by=None):
220
+        """Return a query to collection's objects and ContentRange object
221
+
222
+        Return a filtered and paged query, according to the provided pagination
223
+        (see api.v1.handlers.base.Pagination)
224
+        Also return ContentRange - object with index of first element, last
225
+        element and total count of elements in query(after filtering), and
226
+        a 'valid' parameter to indicate that the paging scope (limit + offset)
227
+        is valid or not (resulted in no data while there was data to provide)
228
+
229
+        :param pagination: Pagination object
230
+        :param filter_by: dict to filter objects {field1: value1, ...}
231
+        :returns: SQLAlchemy query and ContentRange object
232
+        """
233
+        query = cls.all()
234
+        content_range = None
235
+        if filter_by:
236
+            query = query.filter_by(**filter_by)
237
+        query_full = query
238
+        if pagination:
239
+            if pagination.limit > 0 or pagination.limit is None:
240
+                if pagination.order_by:
241
+                    query = cls.order_by(query, pagination.order_by)
242
+                if pagination.offset:
243
+                    query = query.offset(pagination.offset)
244
+                if pagination.limit > 0:
245
+                    query = query.limit(pagination.limit)
246
+            else:
247
+                # making an empty result
248
+                query = query.filter(False)
249
+            if pagination.offset or pagination.limit is not None:
250
+                total = query_full.count()
251
+                selected = query.count() if pagination.limit != 0 else 0
252
+                # first element index=1
253
+                first = pagination.offset + 1 if pagination.offset else 1
254
+                if selected == 0 or pagination.limit == 0:
255
+                    # no data, report first and last as 0
256
+                    first = last = 0
257
+                elif pagination.limit > 0:
258
+                        last = min(first + pagination.limit - 1, total)
259
+                else:
260
+                    last = total
261
+                valid = selected > 0 or pagination.limit == 0 or total == 0
262
+                content_range = cls.content_range(first, last, total, valid)
263
+        return query, content_range
264
+
194 265
     @classmethod
195 266
     def _query_order_by(cls, query, order_by):
196 267
         """Adds order by clause into SQLAlchemy query
@@ -235,6 +306,23 @@ class NailgunCollection(object):
235 306
                                                      order_by=order_by_fields))
236 307
         return sorted(iterable, key=key)
237 308
 
309
+    @classmethod
310
+    def get_iterable(cls, iterable, require=True):
311
+        """Return either iterable or cls.all() when possible
312
+
313
+        :param iterable: model objects collection
314
+        :returns: original iterable or an SQLAlchemy query
315
+        """
316
+        if iterable is not None:
317
+            if cls._is_iterable(iterable) or cls._is_query(iterable):
318
+                return iterable
319
+            else:
320
+                raise TypeError("'%s' object is not iterable" % type(iterable))
321
+        elif cls.single.model:
322
+            return cls.all()
323
+        elif require:
324
+            raise ValueError('iterable not provided and single.model not set')
325
+
238 326
     @classmethod
239 327
     def order_by(cls, iterable, order_by):
240 328
         """Order given iterable by specified order_by.
@@ -244,15 +332,17 @@ class NailgunCollection(object):
244 332
             ORDER BY criterion to SQLAlchemy query. If name starts with '-'
245 333
             desc ordering applies, else asc.
246 334
         :type order_by: tuple of strings or string
335
+        :returns: ordered iterable (SQLAlchemy query)
247 336
         """
248
-        if iterable is None or not order_by:
337
+        if not iterable or not order_by:
249 338
             return iterable
339
+        use_iterable = cls.get_iterable(iterable)
250 340
         if not isinstance(order_by, (list, tuple)):
251 341
             order_by = (order_by,)
252
-        if cls._is_query(iterable):
253
-            return cls._query_order_by(iterable, order_by)
254
-        else:
255
-            return cls._iterable_order_by(iterable, order_by)
342
+        if cls._is_query(use_iterable):
343
+            return cls._query_order_by(use_iterable, order_by)
344
+        elif cls._is_iterable(use_iterable):
345
+            return cls._iterable_order_by(use_iterable, order_by)
256 346
 
257 347
     @classmethod
258 348
     def filter_by(cls, iterable, **kwargs):
@@ -266,10 +356,7 @@ class NailgunCollection(object):
266 356
             else asc.
267 357
         :returns: filtered iterable (SQLAlchemy query)
268 358
         """
269
-        if iterable is not None:
270
-            use_iterable = iterable
271
-        else:
272
-            use_iterable = cls.all()
359
+        use_iterable = cls.get_iterable(iterable)
273 360
         if cls._is_query(use_iterable):
274 361
             return use_iterable.filter_by(**kwargs)
275 362
         elif cls._is_iterable(use_iterable):
@@ -291,7 +378,7 @@ class NailgunCollection(object):
291 378
         :param iterable: iterable (SQLAlchemy query)
292 379
         :returns: filtered iterable (SQLAlchemy query)
293 380
         """
294
-        use_iterable = iterable or cls.all()
381
+        use_iterable = cls.get_iterable(iterable)
295 382
         if cls._is_query(use_iterable):
296 383
             conditions = []
297 384
             for key, value in six.iteritems(kwargs):
@@ -306,8 +393,6 @@ class NailgunCollection(object):
306 393
                 ),
307 394
                 use_iterable
308 395
             )
309
-        else:
310
-            raise TypeError("First argument should be iterable")
311 396
 
312 397
     @classmethod
313 398
     def lock_for_update(cls, iterable):
@@ -318,15 +403,13 @@ class NailgunCollection(object):
318 403
         :param iterable: iterable (SQLAlchemy query)
319 404
         :returns: filtered iterable (SQLAlchemy query)
320 405
         """
321
-        use_iterable = iterable or cls.all()
406
+        use_iterable = cls.get_iterable(iterable)
322 407
         if cls._is_query(use_iterable):
323 408
             return use_iterable.with_lockmode('update')
324 409
         elif cls._is_iterable(use_iterable):
325 410
             # we can't lock abstract iterable, so returning as is
326 411
             # for compatibility
327 412
             return use_iterable
328
-        else:
329
-            raise TypeError("First argument should be iterable")
330 413
 
331 414
     @classmethod
332 415
     def filter_by_list(cls, iterable, field_name, list_of_values,
@@ -341,7 +424,7 @@ class NailgunCollection(object):
341 424
         :returns: filtered iterable (SQLAlchemy query)
342 425
         """
343 426
         field_getter = operator.attrgetter(field_name)
344
-        use_iterable = iterable or cls.all()
427
+        use_iterable = cls.get_iterable(iterable)
345 428
         if cls._is_query(use_iterable):
346 429
             result = use_iterable.filter(
347 430
                 field_getter(cls.single.model).in_(list_of_values)
@@ -353,8 +436,6 @@ class NailgunCollection(object):
353 436
                 lambda i: field_getter(i) in list_of_values,
354 437
                 use_iterable
355 438
             )
356
-        else:
357
-            raise TypeError("First argument should be iterable")
358 439
 
359 440
     @classmethod
360 441
     def filter_by_id_list(cls, iterable, uid_list):
@@ -382,7 +463,7 @@ class NailgunCollection(object):
382 463
         :param options: list of sqlalchemy eagerload types
383 464
         :returns: iterable (SQLAlchemy query)
384 465
         """
385
-        use_iterable = iterable or cls.all()
466
+        use_iterable = cls.get_iterable(iterable)
386 467
         if options:
387 468
             return use_iterable.options(*options)
388 469
         return use_iterable
@@ -404,13 +485,11 @@ class NailgunCollection(object):
404 485
 
405 486
     @classmethod
406 487
     def count(cls, iterable=None):
407
-        use_iterable = iterable or cls.all()
488
+        use_iterable = cls.get_iterable(iterable)
408 489
         if cls._is_query(use_iterable):
409 490
             return use_iterable.count()
410 491
         elif cls._is_iterable(use_iterable):
411 492
             return len(list(iterable))
412
-        else:
413
-            raise TypeError("First argument should be iterable")
414 493
 
415 494
     @classmethod
416 495
     def to_list(cls, iterable=None, fields=None, serializer=None):
@@ -423,7 +502,7 @@ class NailgunCollection(object):
423 502
         :param serializer: the custom serializer
424 503
         :returns: collection of objects as a list of dicts
425 504
         """
426
-        use_iterable = cls.all() if iterable is None else iterable
505
+        use_iterable = cls.get_iterable(iterable)
427 506
         return [
428 507
             cls.single.to_dict(o, fields=fields, serializer=serializer)
429 508
             for o in use_iterable
@@ -465,7 +544,7 @@ class NailgunCollection(object):
465 544
         :param options: list of sqlalchemy mapper options
466 545
         :returns: iterable (SQLAlchemy query)
467 546
         """
468
-        use_iterable = iterable or cls.all()
547
+        use_iterable = cls.get_iterable(iterable)
469 548
         if args:
470 549
             return use_iterable.options(*args)
471 550
         return use_iterable

+ 90
- 0
nailgun/nailgun/test/unit/test_handlers.py View File

@@ -20,9 +20,13 @@ import urllib
20 20
 import web
21 21
 
22 22
 from nailgun.api.v1.handlers.base import BaseHandler
23
+from nailgun.api.v1.handlers.base import CollectionHandler
23 24
 from nailgun.api.v1.handlers.base import handle_errors
25
+from nailgun.api.v1.handlers.base import Pagination
24 26
 from nailgun.api.v1.handlers.base import serialize
25 27
 
28
+from nailgun.objects import NodeCollection
29
+
26 30
 from nailgun.test.base import BaseIntegrationTest
27 31
 from nailgun.utils import reverse
28 32
 
@@ -185,3 +189,89 @@ class TestHandlers(BaseIntegrationTest):
185 189
 
186 190
     def test_get_requested_default(self):
187 191
         self.check_get_requested_mime({}, 'application/json')
192
+
193
+    def test_pagination_class(self):
194
+        # test empty query
195
+        web.ctx.env = {'REQUEST_METHOD': 'GET'}
196
+        pagination = Pagination()
197
+        self.assertEqual(pagination.limit, None)
198
+        self.assertEqual(pagination.offset, None)
199
+        self.assertEqual(pagination.order_by, None)
200
+        # test value retrieval from web + order_by cleanup
201
+        q = 'limit=1&offset=5&order_by=-id, timestamp ,   somefield '
202
+        web.ctx.env['QUERY_STRING'] = q
203
+        pagination = Pagination()
204
+        self.assertEqual(pagination.limit, 1)
205
+        self.assertEqual(pagination.offset, 5)
206
+        self.assertEqual(set(pagination.order_by),
207
+                         set(['-id', 'timestamp', 'somefield']))
208
+        # test incorrect values raise 400
209
+        web.ctx.env['QUERY_STRING'] = 'limit=qwe'
210
+        self.assertRaises(web.HTTPError, Pagination)
211
+        web.ctx.env['QUERY_STRING'] = 'offset=asd'
212
+        self.assertRaises(web.HTTPError, Pagination)
213
+        web.ctx.env['QUERY_STRING'] = 'limit='
214
+        self.assertRaises(web.HTTPError, Pagination)
215
+        web.ctx.env['QUERY_STRING'] = 'offset=-2'
216
+        self.assertRaises(web.HTTPError, Pagination)
217
+        # test constructor, limit = 0 -> 0, offset '0' -> 0, bad order_by
218
+        pagination = Pagination(0, '0', ', ,,,  ,')
219
+        self.assertEqual(pagination.limit, 0)
220
+        self.assertEqual(pagination.offset, 0)
221
+        self.assertEqual(pagination.order_by, None)
222
+
223
+    def test_pagination_of_node_collection(self):
224
+        def assert_pagination_and_cont_rng(q, cr, sz, first, last, ttl, valid):
225
+            self.assertEqual(q.count(), sz)
226
+            self.assertEqual(cr.first, first)
227
+            self.assertEqual(cr.last, last)
228
+            self.assertEqual(cr.total, ttl)
229
+            self.assertEqual(cr.valid, valid)
230
+
231
+        self.env.create_nodes(5)
232
+        # test pagination limited to 2 first items
233
+        pagination = Pagination(limit=2)
234
+        q, cr = NodeCollection.scope(pagination)
235
+        assert_pagination_and_cont_rng(q, cr, 2, 1, 2, 5, True)
236
+        # test invalid pagination
237
+        pagination = Pagination(offset=5)
238
+        q, cr = NodeCollection.scope(pagination)
239
+        assert_pagination_and_cont_rng(q, cr, 0, 0, 0, 5, False)
240
+        # test limit=0, offset ignored
241
+        pagination = Pagination(limit=0, offset=999)
242
+        q, cr = NodeCollection.scope(pagination)
243
+        assert_pagination_and_cont_rng(q, cr, 0, 0, 0, 5, True)
244
+        # test limit+offset+order_by
245
+        pagination = Pagination(limit=3, offset=1, order_by='-id')
246
+        q, cr = NodeCollection.scope(pagination)
247
+        assert_pagination_and_cont_rng(q, cr, 3, 2, 4, 5, True)
248
+        ids = sorted([i.id for i in self.env.nodes])
249
+        n = q.all()
250
+        self.assertEqual(n[0].id, ids[3])
251
+        self.assertEqual(n[1].id, ids[2])
252
+        self.assertEqual(n[2].id, ids[1])
253
+
254
+    def test_collection_handler(self):
255
+        FakeHandler = CollectionHandler
256
+        # setting a collection is mandatory, CollectionHandler is not ready
257
+        # to use "as-is"
258
+        FakeHandler.collection = NodeCollection
259
+        urls = ("/collection_test", "collection_test")
260
+        app = web.application(urls, {'collection_test': FakeHandler})
261
+        resp = app.request(urls[0])
262
+        self.assertEqual(resp.status, '200 OK')
263
+
264
+    def test_content_range_header(self):
265
+        self.env.create_nodes(5)
266
+        FakeHandler = CollectionHandler
267
+        FakeHandler.collection = NodeCollection
268
+        urls = ("/collection_test", "collection_test")
269
+        app = web.application(urls, {'collection_test': FakeHandler})
270
+        # test paginated query
271
+        resp = app.request("/collection_test?limit=3&offset=1")
272
+        self.assertEqual(resp.status, '200 OK')
273
+        self.assertIn('Content-Range', resp.headers)
274
+        self.assertEqual(resp.headers['Content-Range'], 'objects 2-4/5')
275
+        # test invalid range (offset = 6 >= number of nodes ---> no data)
276
+        resp = app.request("/collection_test?limit=3&offset=5&order_by=id")
277
+        self.assertEqual(resp.status, '416 Range Not Satisfiable')

Loading…
Cancel
Save