Merge "Support handling large queries for vnf packages"

tacker/api/vnfpkgm/v1/controller.py

@@ -25,8 +25,10 @@ from zipfile import ZipFile
 from glance_store import exceptions as store_exceptions
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_serialization import jsonutils
 from oslo_utils import encodeutils
 from oslo_utils import excutils
+from oslo_utils import timeutils
 from oslo_utils import uuidutils
 
 from tacker._i18n import _
@@ -57,6 +59,7 @@ class VnfPkgmController(wsgi.Controller):
         super(VnfPkgmController, self).__init__()
         self.rpc_api = vnf_pkgm_rpc.VNFPackageRPCAPI()
         glance_store.initialize_glance_store()
+        self._nextpages = {}
 
     def _get_vnf_package(self, id, request):
         # check if id is of type uuid format
@@ -72,6 +75,13 @@ class VnfPkgmController(wsgi.Controller):
             raise webob.exc.HTTPNotFound(explanation=msg)
         return vnf_package
 
+    def _delete_expired_nextpages(self, nextpages):
+        for k, v in nextpages.items():
+            if timeutils.is_older_than(v['created_time'],
+                    CONF.vnf_package.nextpage_expiration_time):
+                LOG.debug('Old nextpages are deleted. id: %s' % k)
+                nextpages.pop(k)
+
     @wsgi.response(http_client.CREATED)
     @wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN))
     @validation.schema(vnf_packages.create)
@@ -120,8 +130,9 @@ class VnfPkgmController(wsgi.Controller):
     @wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN))
     @validation.query_schema(vnf_packages.query_params_v1)
     def index(self, request):
-        context = request.environ['tacker.context']
+        if 'tacker.context' in request.environ:
-        context.can(vnf_package_policies.VNFPKGM % 'index')
+            context = request.environ['tacker.context']
+            context.can(vnf_package_policies.VNFPKGM % 'index')
 
         search_opts = {}
         search_opts.update(request.GET)
@@ -139,6 +150,8 @@ class VnfPkgmController(wsgi.Controller):
         fields = request.GET.get('fields')
         exclude_fields = request.GET.get('exclude_fields')
         filters = request.GET.get('filter')
+        nextpage = request.GET.get('nextpage_opaque_marker')
+        allrecords = request.GET.get('all_records')
         if not (all_fields or fields or exclude_fields):
             exclude_default = True
 
@@ -148,14 +161,45 @@ class VnfPkgmController(wsgi.Controller):
 
         filters = self._view_builder.validate_filter(filters)
 
-        vnf_packages = vnf_package_obj.VnfPackagesList.get_by_filters(
+        results = []
-            request.context, read_deleted='no', filters=filters)
 
-        return self._view_builder.index(vnf_packages,
+        if allrecords != 'yes' and nextpage:
-                                        all_fields=all_fields,
+            self._delete_expired_nextpages(self._nextpages)
-                                        exclude_fields=exclude_fields,
+
-                                        fields=fields,
+            if nextpage in self._nextpages:
-                                        exclude_default=exclude_default)
+                results = self._nextpages.pop(
+                    nextpage)['nextpage']
+        else:
+            vnf_packages = vnf_package_obj.VnfPackagesList.get_by_filters(
+                request.context, read_deleted='no', filters=filters)
+
+            results = self._view_builder.index(vnf_packages,
+                                               all_fields=all_fields,
+                                               exclude_fields=exclude_fields,
+                                               fields=fields,
+                                               exclude_default=exclude_default)
+
+        res = webob.Response(content_type='application/json')
+        res.status_int = 200
+
+        if (allrecords != 'yes' and
+                len(results) > CONF.vnf_package.vnf_package_num):
+            nextpageid = uuidutils.generate_uuid()
+            links = ('Link', '<%s?nextpage_opaque_marker=%s>; rel="next"' % (
+                request.path_url, nextpageid))
+            res.headerlist.append(links)
+            res.body = jsonutils.dump_as_bytes(
+                results[: CONF.vnf_package.vnf_package_num], default=str)
+
+            self._delete_expired_nextpages(self._nextpages)
+
+            remain = results[CONF.vnf_package.vnf_package_num:]
+            self._nextpages.update({nextpageid:
+                {'created_time': timeutils.utcnow(), 'nextpage': remain}})
+        else:
+            res.body = jsonutils.dump_as_bytes(results, default=str)
+
+        return res
 
     @wsgi.response(http_client.NO_CONTENT)
     @wsgi.expected_errors((http_client.FORBIDDEN, http_client.NOT_FOUND,

tacker/conf/vnf_package.py

@@ -76,6 +76,12 @@ Related options:
                 'provider', 'product_name', 'software_version',
                 'vnfm_info', 'flavour_id', 'flavour_description'],
                 help=_("List of del inputs from lower-vnfd")),
+    cfg.IntOpt('vnf_package_num',
+               default=100,
+               help=_("Number of vnf_packages contained in 1 page")),
+    cfg.IntOpt('nextpage_expiration_time',
+               default=3600,
+               help=_("Expiration time (sec) for paging")),
 
 ]
 

tacker/tests/unit/vnfpkgm/test_controller.py

@@ -18,10 +18,12 @@ import ddt
 from http import client as http_client
 import json
 import os
+import re
 from unittest import mock
 import urllib
 from webob import exc
 
+from oslo_config import cfg
 from oslo_serialization import jsonutils
 
 from tacker.api.vnfpkgm.v1 import controller
@@ -127,7 +129,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_attribute_selector_all_fields(self, mock_vnf_list):
@@ -138,7 +142,9 @@ class TestController(base.TestCase):
         mock_vnf_list.return_value = fakes.return_vnf_package_list()
         res_dict = self.controller.index(req)
         expected_result = fakes.index_response()
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_attribute_selector_exclude_default(self, mock_vnf_list):
@@ -154,7 +160,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -172,7 +180,9 @@ class TestController(base.TestCase):
         res_dict = self.controller.index(req)
         remove_attrs = [params['exclude_fields']]
         expected_result = fakes.index_response(remove_attrs=remove_attrs)
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -199,7 +209,9 @@ class TestController(base.TestCase):
         res_dict = self.controller.index(req)
         remove_attrs = [x for x in complex_attrs if x != params['fields']]
         expected_result = fakes.index_response(remove_attrs=remove_attrs)
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_attribute_selector_user_defined_data_combination(self,
@@ -226,7 +238,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'additionalArtifacts'],
             vnf_package_updates=vnf_package_updates)
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_attribute_selector_user_defined_data(self, mock_vnf_list):
@@ -238,7 +252,9 @@ class TestController(base.TestCase):
         res_dict = self.controller.index(req)
         expected_result = fakes.index_response(remove_attrs=[
             'checksum', 'softwareImages', 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_attribute_selector_nested_complex_attribute(self,
@@ -264,7 +280,9 @@ class TestController(base.TestCase):
         expected_result = fakes.index_response(remove_attrs=[
             'checksum', 'userDefinedData'],
             vnf_package_updates=vnf_package_updates)
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -303,7 +321,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     def test_index_filter_combination(self, mock_vnf_list):
@@ -321,7 +341,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -368,7 +390,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -400,7 +424,9 @@ class TestController(base.TestCase):
                 'checksum',
                 'userDefinedData',
                 'additionalArtifacts'])
-        self.assertEqual(expected_result, res_dict)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
 
     @mock.patch.object(VnfPackagesList, "get_by_filters")
     @ddt.data(
@@ -534,6 +560,52 @@ class TestController(base.TestCase):
         self.assertRaises(tacker_exc.ValidationError, self.controller.index,
                           req)
 
+    @mock.patch.object(VnfPackagesList, "get_by_filters")
+    @ddt.data(
+        {'params': {'all_records': 'yes'},
+            'result_names': ['sample1', 'sample2', 'sample3', 'sample4']},
+        {'params': {'all_records': 'yes', 'nextpage_opaque_marker': 'abc'},
+            'result_names': ['sample1', 'sample2', 'sample3', 'sample4']},
+        {'params': {'nextpage_opaque_marker': 'abc'},
+            'result_names': []},
+        {'params': {},
+            'result_names': ['sample2']}
+    )
+    def test_index_paging(self, values, mock_vnf_list):
+        cfg.CONF.set_override('vnf_package_num', 1, group='vnf_package')
+        query = urllib.parse.urlencode(values['params'])
+        req = fake_request.HTTPRequest.blank('/vnfpkgm/v1/vnf_packages?' +
+            query)
+        mock_vnf_list.return_value = [
+            fakes.return_vnfpkg_obj(
+                vnfd_updates={'vnf_product_name': 'sample1'}),
+            fakes.return_vnfpkg_obj(
+                vnfd_updates={'vnf_product_name': 'sample2'}),
+            fakes.return_vnfpkg_obj(
+                vnfd_updates={'vnf_product_name': 'sample3'}),
+            fakes.return_vnfpkg_obj(
+                vnfd_updates={'vnf_product_name': 'sample4'})
+        ]
+        expected_result = []
+        for name in values['result_names']:
+            expected_result += fakes.index_response(
+                remove_attrs=[
+                    'softwareImages',
+                    'checksum',
+                    'userDefinedData',
+                    'additionalArtifacts'],
+                vnf_package_updates={'vnfProductName': name})
+        res_dict = self.controller.index(req)
+        if 'Link' in res_dict.headers:
+            next_url = re.findall('<(.*)>', res_dict.headers['Link'])[0]
+            query = urllib.parse.urlparse(next_url).query
+            req = fake_request.HTTPRequest.blank('/vnfpkgm/v1/vnf_packages?' +
+                query)
+            res_dict = self.controller.index(req)
+        self.assertEqual(
+            jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
+                default=str)), res_dict.json)
+
     @mock.patch.object(vnf_package.VnfPackage, "get_by_id")
     @mock.patch.object(VNFPackageRPCAPI, "delete_vnf_package")
     def test_delete_with_204_status(self, mock_delete_rpc, mock_vnf_by_id):