openstack
/
tacker
Code Issues Proposed changes

Merge "Support handling of large query results"

This commit is contained in:
Zuul 2022-05-20 10:10:39 +00:00 committed by Gerrit Code Review
parent 62a91ea41f e0e7b538ee
commit 7eb143eeb8
10 changed files with 164 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tacker/sol_refactored/api/wsgi.py
View File

@ -65,7 +65,7 @@ class SolResponse(object):
self.body = body
self.headers = {}
for hdr in self.allowed_headers:
if hdr in kwargs:
if kwargs.get(hdr):
self.headers[hdr] = kwargs[hdr]
self.headers.setdefault('version', api_version.CURRENT_VERSION)
self.headers.setdefault('accept-ranges', 'none')

12
tacker/sol_refactored/common/config.py
View File

@ -34,6 +34,18 @@ VNFM_OPTS = [
cfg.IntOpt('kubernetes_vim_rsc_wait_timeout',
default=500,
help='Timeout (second) of k8s res creation.'),
cfg.IntOpt('vnf_instance_page_size',
default=0, # 0 means no paging
help=_('Paged response size of the query result '
'for VNF instances.')),
cfg.IntOpt('subscription_page_size',
default=0, # 0 means no paging
help=_('Paged response size of the query result '
'for Subscriptions.')),
cfg.IntOpt('lcm_op_occ_page_size',
default=0, # 0 means no paging
help=_('Paged response size of the query result '
'for VNF LCM operation occurrences.')),
# NOTE: This is for test use since it is convenient to be able to delete
# under development.
cfg.BoolOpt('test_enable_lcm_op_occ_delete',

4
tacker/sol_refactored/common/exceptions.py
View File

@ -347,3 +347,7 @@ class NotSupportedContentType(SolHttpError415):
class MalformedRequestBody(SolHttpError400):
message = _("Malformed request body.")
class InvalidPagingMarker(SolHttpError400):
message = _("Paging marker value %(marker)s is invalid.")

4
tacker/sol_refactored/common/lcm_op_occ_utils.py
View File

@ -36,8 +36,8 @@ def get_lcmocc(context, lcmocc_id):
return lcmocc
def get_lcmocc_all(context):
return objects.VnfLcmOpOccV2.get_all(context)
def get_lcmocc_all(context, marker=None):
return objects.VnfLcmOpOccV2.get_all(context, marker)
def lcmocc_href(lcmocc_id, endpoint):

4
tacker/sol_refactored/common/subscription_utils.py
View File

@ -42,8 +42,8 @@ def get_subsc(context, subsc_id):
return subsc
def get_subsc_all(context):
return objects.LccnSubscriptionV2.get_all(context)
def get_subsc_all(context, marker=None):
return objects.LccnSubscriptionV2.get_all(context, marker)
def subsc_href(subsc_id, endpoint):

4
tacker/sol_refactored/common/vnf_instance_utils.py
View File

@ -30,8 +30,8 @@ def get_inst(context, inst_id):
return inst
def get_inst_all(context):
return objects.VnfInstanceV2.get_all(context)
def get_inst_all(context, marker=None):
return objects.VnfInstanceV2.get_all(context, marker)
def inst_href(inst_id, endpoint):

29
tacker/sol_refactored/controller/vnflcm_v2.py
View File

@ -118,11 +118,15 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController):
selector = self._inst_view.parse_selector(request.GET)
insts = inst_utils.get_inst_all(request.context)
pager = self._inst_view.parse_pager(request)
resp_body = self._inst_view.detail_list(insts, filters, selector)
insts = inst_utils.get_inst_all(request.context,
marker=pager.marker)
return sol_wsgi.SolResponse(200, resp_body)
resp_body = self._inst_view.detail_list(insts, filters,
selector, pager)
return sol_wsgi.SolResponse(200, resp_body, link=pager.get_link())
def show(self, request, id):
inst = inst_utils.get_inst(request.context, id)
@ -502,11 +506,14 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController):
else:
filters = None
subscs = subsc_utils.get_subsc_all(request.context)
pager = self._subsc_view.parse_pager(request)
resp_body = self._subsc_view.detail_list(subscs, filters)
subscs = subsc_utils.get_subsc_all(request.context,
marker=pager.marker)
return sol_wsgi.SolResponse(200, resp_body)
resp_body = self._subsc_view.detail_list(subscs, filters, pager)
return sol_wsgi.SolResponse(200, resp_body, link=pager.get_link())
def subscription_show(self, request, id):
subsc = subsc_utils.get_subsc(request.context, id)
@ -532,11 +539,15 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController):
selector = self._lcmocc_view.parse_selector(request.GET)
lcmoccs = lcmocc_utils.get_lcmocc_all(request.context)
pager = self._lcmocc_view.parse_pager(request)
resp_body = self._lcmocc_view.detail_list(lcmoccs, filters, selector)
lcmoccs = lcmocc_utils.get_lcmocc_all(request.context,
marker=pager.marker)
return sol_wsgi.SolResponse(200, resp_body)
resp_body = self._lcmocc_view.detail_list(lcmoccs, filters,
selector, pager)
return sol_wsgi.SolResponse(200, resp_body, link=pager.get_link())
def lcm_op_occ_show(self, request, id):
lcmocc = lcmocc_utils.get_lcmocc(request.context, id)

79
tacker/sol_refactored/controller/vnflcm_view.py
View File

@ -20,6 +20,7 @@ from dateutil import parser
from oslo_log import log as logging
from tacker.sol_refactored.common import config
from tacker.sol_refactored.common import exceptions as sol_ex
from tacker.sol_refactored.common import lcm_op_occ_utils as lcmocc_utils
from tacker.sol_refactored.common import subscription_utils as subsc_utils
@ -28,6 +29,7 @@ from tacker.sol_refactored import objects
LOG = logging.getLogger(__name__)
CONF = config.CONF
class KeyAttribute(object):
@ -308,9 +310,27 @@ class BaseViewBuilder(object):
return False
return True
def detail_list(self, values, filters, selector):
return [self.detail(v, selector) for v in values
if self.match_filters(v, filters)]
def parse_pager(self, request, page_size):
"""Implement SOL013 5.4 Handling of large query results"""
marker = request.GET.get('nextpage_opaque_marker')
req_url = request.url
return Pager(marker, req_url, page_size)
def detail_list(self, values, filters, selector, pager):
resp_body = [self.detail(v, selector) for v in values
if self.match_filters(v, filters)]
if pager.page_size == 0:
pager.next_marker = None
return resp_body
if len(resp_body) > pager.page_size:
resp_body = resp_body[:pager.page_size]
pager.next_marker = resp_body[-1]["id"]
else:
pager.next_marker = None
return resp_body
class InstanceViewBuilder(BaseViewBuilder):
@ -326,6 +346,10 @@ class InstanceViewBuilder(BaseViewBuilder):
def parse_filter(self, filter):
return super().parse_filter(filter)
def parse_pager(self, request):
page_size = CONF.v2_vnfm.vnf_instance_page_size
return super().parse_pager(request, page_size)
def detail(self, inst, selector=None):
# NOTE: _links is not saved in DB. create when it is necessary.
if not inst.obj_attr_is_set('_links'):
@ -344,8 +368,8 @@ class InstanceViewBuilder(BaseViewBuilder):
resp = selector.filter(inst, resp)
return resp
def detail_list(self, insts, filters, selector):
return super().detail_list(insts, filters, selector)
def detail_list(self, insts, filters, selector, pager):
return super().detail_list(insts, filters, selector, pager)
class LcmOpOccViewBuilder(BaseViewBuilder):
@ -361,6 +385,10 @@ class LcmOpOccViewBuilder(BaseViewBuilder):
def parse_filter(self, filter):
return super().parse_filter(filter)
def parse_pager(self, request):
page_size = CONF.v2_vnfm.lcm_op_occ_page_size
return super().parse_pager(request, page_size)
def detail(self, lcmocc, selector=None):
# NOTE: _links is not saved in DB. create when it is necessary.
if not lcmocc.obj_attr_is_set('_links'):
@ -371,8 +399,8 @@ class LcmOpOccViewBuilder(BaseViewBuilder):
resp = selector.filter(lcmocc, resp)
return resp
def detail_list(self, lcmoccs, filters, selector):
return super().detail_list(lcmoccs, filters, selector)
def detail_list(self, lcmoccs, filters, selector, pager):
return super().detail_list(lcmoccs, filters, selector, pager)
class SubscriptionViewBuilder(BaseViewBuilder):
@ -382,6 +410,10 @@ class SubscriptionViewBuilder(BaseViewBuilder):
def parse_filter(self, filter):
return super().parse_filter(filter)
def parse_pager(self, request):
page_size = CONF.v2_vnfm.subscription_page_size
return super().parse_pager(request, page_size)
def detail(self, subsc, selector=None):
# NOTE: _links is not saved in DB. create when it is necessary.
if not subsc.obj_attr_is_set('_links'):
@ -398,5 +430,34 @@ class SubscriptionViewBuilder(BaseViewBuilder):
resp = selector.filter(subsc, resp)
return resp
def detail_list(self, subscs, filters):
return super().detail_list(subscs, filters, None)
def detail_list(self, subscs, filters, pager):
return super().detail_list(subscs, filters, None, pager)
class Pager:
def __init__(self, marker, req_url, page_size):
self.marker = marker
self.req_url = req_url
self.page_size = page_size
self.next_marker = None
def _marker_string(self, marker):
return f'nextpage_opaque_marker={marker}'
def _link_value(self, url):
return f'<{url}>;rel="next"'
def get_link(self):
if self.next_marker is None:
return
if self.marker is not None:
# req_url includes marker string
url = self.req_url.replace(self._marker_string(self.marker),
self._marker_string(self.next_marker))
elif '?' not in self.req_url:
url = f'{self.req_url}?{self._marker_string(self.next_marker)}'
else:
url = f'{self.req_url}&{self._marker_string(self.next_marker)}'
return self._link_value(url)

10
tacker/sol_refactored/objects/base.py
View File

@ -18,6 +18,7 @@ import collections
import contextlib
import datetime
from oslo_db.sqlalchemy import utils
from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_serialization import jsonutils
@ -26,6 +27,7 @@ from oslo_versionedobjects import base as ovoo_base
from oslo_versionedobjects import exception as ovoo_exc
from tacker.db import api as db_api
from tacker.sol_refactored.common import exceptions as sol_ex
from tacker.sol_refactored.db.sqlalchemy import models
from tacker.sol_refactored import objects
from tacker.sol_refactored.objects import fields as obj_fields
@ -376,9 +378,15 @@ class TackerPersistentObject(TackerObject):
@classmethod
@db_api.context_manager.reader
def get_all(cls, context):
def get_all(cls, context, marker=None):
model_cls = getattr(models, cls.__name__)
query = context.session.query(model_cls)
if marker is not None:
db_obj = query.filter(model_cls.id == marker).one_or_none()
if db_obj is None:
raise sol_ex.InvalidPagingMarker(marker=marker)
query = utils.paginate_query(query, model_cls, None, ['id'],
marker=db_obj)
result = query.all()
return [cls.from_db_obj(item) for item in result]

42
tacker/tests/unit/sol_refactored/controller/test_vnflcm_view.py
View File

@ -16,6 +16,7 @@
from unittest import mock
from dateutil import parser
import ddt
from tacker.sol_refactored.common import exceptions as sol_ex
from tacker.sol_refactored.controller import vnflcm_view
@ -184,3 +185,44 @@ class TestBaseViewBuilder(base.BaseTestCase):
self.assertRaises(sol_ex.InvalidAttributeFilter,
builder.parse_filter,
"(gt,foo,1,2)")
@ddt.ddt
class TestPager(base.BaseTestCase):
@ddt.data(
{'marker': '5a72eeef-d912-419a-b0e2-23dd3cbbc688',
'req_url': ('http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?nextpage_opaque_marker='
'5a72eeef-d912-419a-b0e2-23dd3cbbc688'),
'next_marker': None,
'expect_link': None},
{'marker': '5a72eeef-d912-419a-b0e2-23dd3cbbc688',
'req_url': ('http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?nextpage_opaque_marker='
'5a72eeef-d912-419a-b0e2-23dd3cbbc688'),
'next_marker': '5a72eeef-d912-419a-b0e2-23dd3cbbc700',
'expect_link': ('<http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?nextpage_opaque_marker='
'5a72eeef-d912-419a-b0e2-23dd3cbbc700>;rel="next"')},
{'marker': None,
'req_url': ('http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?all_fields=1'),
'next_marker': '5a72eeef-d912-419a-b0e2-23dd3cbbc700',
'expect_link': ('<http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?all_fields=1&nextpage_opaque_marker='
'5a72eeef-d912-419a-b0e2-23dd3cbbc700>;rel="next"')},
{'marker': None,
'req_url': 'http://127.0.0.1:9890/vnflcm/v2/vnf_instances',
'next_marker': '5a72eeef-d912-419a-b0e2-23dd3cbbc700',
'expect_link': ('<http://127.0.0.1:9890/vnflcm/v2/vnf_instances'
'?nextpage_opaque_marker='
'5a72eeef-d912-419a-b0e2-23dd3cbbc700>;rel="next"')})
@ddt.unpack
def test_get_link(self, marker, req_url, next_marker, expect_link):
page_size = 10
pager = vnflcm_view.Pager(marker, req_url, page_size)
pager.next_marker = next_marker
link = pager.get_link()
self.assertEqual(expect_link, link)