openstack
/
tacker
Code Issues Proposed changes

Merge "Support handling large queries for vnf packages"

This commit is contained in:
Zuul 2022-09-06 13:51:29 +00:00 committed by Gerrit Code Review
parent 213f5f59ba c5d4b335f3
commit 8bddc176e9
3 changed files with 143 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
tacker/api/vnfpkgm/v1/controller.py
View File

@ -25,8 +25,10 @@ from zipfile import ZipFile
from glance_store import exceptions as store_exceptions
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import timeutils
from oslo_utils import uuidutils
from tacker._i18n import _
@ -57,6 +59,7 @@ class VnfPkgmController(wsgi.Controller):
super(VnfPkgmController, self).__init__()
self.rpc_api = vnf_pkgm_rpc.VNFPackageRPCAPI()
glance_store.initialize_glance_store()
self._nextpages = {}
def _get_vnf_package(self, id, request):
# check if id is of type uuid format
@ -72,6 +75,13 @@ class VnfPkgmController(wsgi.Controller):
raise webob.exc.HTTPNotFound(explanation=msg)
return vnf_package
def _delete_expired_nextpages(self, nextpages):
for k, v in nextpages.items():
if timeutils.is_older_than(v['created_time'],
CONF.vnf_package.nextpage_expiration_time):
LOG.debug('Old nextpages are deleted. id: %s' % k)
nextpages.pop(k)
@wsgi.response(http_client.CREATED)
@wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN))
@validation.schema(vnf_packages.create)
@ -120,8 +130,9 @@ class VnfPkgmController(wsgi.Controller):
@wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN))
@validation.query_schema(vnf_packages.query_params_v1)
def index(self, request):
context = request.environ['tacker.context']
context.can(vnf_package_policies.VNFPKGM % 'index')
if 'tacker.context' in request.environ:
context = request.environ['tacker.context']
context.can(vnf_package_policies.VNFPKGM % 'index')
search_opts = {}
search_opts.update(request.GET)
@ -139,6 +150,8 @@ class VnfPkgmController(wsgi.Controller):
fields = request.GET.get('fields')
exclude_fields = request.GET.get('exclude_fields')
filters = request.GET.get('filter')
nextpage = request.GET.get('nextpage_opaque_marker')
allrecords = request.GET.get('all_records')
if not (all_fields or fields or exclude_fields):
exclude_default = True
@ -148,14 +161,45 @@ class VnfPkgmController(wsgi.Controller):
filters = self._view_builder.validate_filter(filters)
vnf_packages = vnf_package_obj.VnfPackagesList.get_by_filters(
request.context, read_deleted='no', filters=filters)
results = []
return self._view_builder.index(vnf_packages,
all_fields=all_fields,
exclude_fields=exclude_fields,
fields=fields,
exclude_default=exclude_default)
if allrecords != 'yes' and nextpage:
self._delete_expired_nextpages(self._nextpages)
if nextpage in self._nextpages:
results = self._nextpages.pop(
nextpage)['nextpage']
else:
vnf_packages = vnf_package_obj.VnfPackagesList.get_by_filters(
request.context, read_deleted='no', filters=filters)
results = self._view_builder.index(vnf_packages,
all_fields=all_fields,
exclude_fields=exclude_fields,
fields=fields,
exclude_default=exclude_default)
res = webob.Response(content_type='application/json')
res.status_int = 200
if (allrecords != 'yes' and
len(results) > CONF.vnf_package.vnf_package_num):
nextpageid = uuidutils.generate_uuid()
links = ('Link', '<%s?nextpage_opaque_marker=%s>; rel="next"' % (
request.path_url, nextpageid))
res.headerlist.append(links)
res.body = jsonutils.dump_as_bytes(
results[: CONF.vnf_package.vnf_package_num], default=str)
self._delete_expired_nextpages(self._nextpages)
remain = results[CONF.vnf_package.vnf_package_num:]
self._nextpages.update({nextpageid:
{'created_time': timeutils.utcnow(), 'nextpage': remain}})
else:
res.body = jsonutils.dump_as_bytes(results, default=str)
return res
@wsgi.response(http_client.NO_CONTENT)
@wsgi.expected_errors((http_client.FORBIDDEN, http_client.NOT_FOUND,

6
tacker/conf/vnf_package.py
View File

@ -76,6 +76,12 @@ Related options:
'provider', 'product_name', 'software_version',
'vnfm_info', 'flavour_id', 'flavour_description'],
help=_("List of del inputs from lower-vnfd")),
cfg.IntOpt('vnf_package_num',
default=100,
help=_("Number of vnf_packages contained in 1 page")),
cfg.IntOpt('nextpage_expiration_time',
default=3600,
help=_("Expiration time (sec) for paging")),
]

96
tacker/tests/unit/vnfpkgm/test_controller.py
View File

@ -18,10 +18,12 @@ import ddt
from http import client as http_client
import json
import os
import re
from unittest import mock
import urllib
from webob import exc
from oslo_config import cfg
from oslo_serialization import jsonutils
from tacker.api.vnfpkgm.v1 import controller
@ -127,7 +129,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_attribute_selector_all_fields(self, mock_vnf_list):
@ -138,7 +142,9 @@ class TestController(base.TestCase):
mock_vnf_list.return_value = fakes.return_vnf_package_list()
res_dict = self.controller.index(req)
expected_result = fakes.index_response()
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_attribute_selector_exclude_default(self, mock_vnf_list):
@ -154,7 +160,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -172,7 +180,9 @@ class TestController(base.TestCase):
res_dict = self.controller.index(req)
remove_attrs = [params['exclude_fields']]
expected_result = fakes.index_response(remove_attrs=remove_attrs)
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -199,7 +209,9 @@ class TestController(base.TestCase):
res_dict = self.controller.index(req)
remove_attrs = [x for x in complex_attrs if x != params['fields']]
expected_result = fakes.index_response(remove_attrs=remove_attrs)
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_attribute_selector_user_defined_data_combination(self,
@ -226,7 +238,9 @@ class TestController(base.TestCase):
'checksum',
'additionalArtifacts'],
vnf_package_updates=vnf_package_updates)
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_attribute_selector_user_defined_data(self, mock_vnf_list):
@ -238,7 +252,9 @@ class TestController(base.TestCase):
res_dict = self.controller.index(req)
expected_result = fakes.index_response(remove_attrs=[
'checksum', 'softwareImages', 'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_attribute_selector_nested_complex_attribute(self,
@ -264,7 +280,9 @@ class TestController(base.TestCase):
expected_result = fakes.index_response(remove_attrs=[
'checksum', 'userDefinedData'],
vnf_package_updates=vnf_package_updates)
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -303,7 +321,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
def test_index_filter_combination(self, mock_vnf_list):
@ -321,7 +341,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -368,7 +390,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -400,7 +424,9 @@ class TestController(base.TestCase):
'checksum',
'userDefinedData',
'additionalArtifacts'])
self.assertEqual(expected_result, res_dict)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
@ -534,6 +560,52 @@ class TestController(base.TestCase):
self.assertRaises(tacker_exc.ValidationError, self.controller.index,
req)
@mock.patch.object(VnfPackagesList, "get_by_filters")
@ddt.data(
{'params': {'all_records': 'yes'},
'result_names': ['sample1', 'sample2', 'sample3', 'sample4']},
{'params': {'all_records': 'yes', 'nextpage_opaque_marker': 'abc'},
'result_names': ['sample1', 'sample2', 'sample3', 'sample4']},
{'params': {'nextpage_opaque_marker': 'abc'},
'result_names': []},
{'params': {},
'result_names': ['sample2']}
)
def test_index_paging(self, values, mock_vnf_list):
cfg.CONF.set_override('vnf_package_num', 1, group='vnf_package')
query = urllib.parse.urlencode(values['params'])
req = fake_request.HTTPRequest.blank('/vnfpkgm/v1/vnf_packages?' +
query)
mock_vnf_list.return_value = [
fakes.return_vnfpkg_obj(
vnfd_updates={'vnf_product_name': 'sample1'}),
fakes.return_vnfpkg_obj(
vnfd_updates={'vnf_product_name': 'sample2'}),
fakes.return_vnfpkg_obj(
vnfd_updates={'vnf_product_name': 'sample3'}),
fakes.return_vnfpkg_obj(
vnfd_updates={'vnf_product_name': 'sample4'})
]
expected_result = []
for name in values['result_names']:
expected_result += fakes.index_response(
remove_attrs=[
'softwareImages',
'checksum',
'userDefinedData',
'additionalArtifacts'],
vnf_package_updates={'vnfProductName': name})
res_dict = self.controller.index(req)
if 'Link' in res_dict.headers:
next_url = re.findall('<(.*)>', res_dict.headers['Link'])[0]
query = urllib.parse.urlparse(next_url).query
req = fake_request.HTTPRequest.blank('/vnfpkgm/v1/vnf_packages?' +
query)
res_dict = self.controller.index(req)
self.assertEqual(
jsonutils.loads(jsonutils.dump_as_bytes(expected_result,
default=str)), res_dict.json)
@mock.patch.object(vnf_package.VnfPackage, "get_by_id")
@mock.patch.object(VNFPackageRPCAPI, "delete_vnf_package")
def test_delete_with_204_status(self, mock_delete_rpc, mock_vnf_by_id):