Merge "Cache Get results for some nsxlib resources"

This commit is contained in:
Zuul 2017-10-02 08:13:10 +00:00 committed by Gerrit Code Review
commit 1ab929cb0a
5 changed files with 171 additions and 9 deletions

View File

@ -252,6 +252,10 @@ class NsxClientTestCase(NsxLibTestCase):
mock_call = getattr(self._record, verb.lower())
mock_call.assert_any_call(**kwargs)
def call_count(self, verb):
mock_call = getattr(self._record, verb.lower())
return mock_call.call_count
@property
def recorded_calls(self):
return self._record

View File

@ -67,6 +67,14 @@ def assert_call(verb, client_or_resource,
'headers': headers, 'cert': None, 'timeout': timeout})
def mock_calls_count(verb, client_or_resource):
nsx_client = client_or_resource
if getattr(nsx_client, 'client', None) is not None:
nsx_client = nsx_client.client
cluster = nsx_client._conn
return cluster.call_count(verb)
def assert_json_call(verb, client_or_resource, url,
verify=nsxlib_testcase.NSX_CERT,
data=None,

View File

@ -15,6 +15,7 @@
#
import copy
import eventlet
import mock
from oslo_serialization import jsonutils
@ -28,6 +29,7 @@ from vmware_nsxlib.v3 import core_resources
from vmware_nsxlib.v3 import exceptions
from vmware_nsxlib.v3 import nsx_constants
from vmware_nsxlib.v3 import resources
from vmware_nsxlib.v3 import utils
class BaseTestResource(nsxlib_testcase.NsxClientTestCase):
@ -1118,6 +1120,10 @@ class TransportZone(BaseTestResource):
tz_type = tz.get_transport_type(fake_tz['id'])
self.assertEqual(tz.TRANSPORT_TYPE_OVERLAY, tz_type)
# call it again to test it when cached
tz_type = tz.get_transport_type(fake_tz['id'])
self.assertEqual(tz.TRANSPORT_TYPE_OVERLAY, tz_type)
def test_get_host_switch_mode(self):
fake_tz = test_constants.FAKE_TZ.copy()
tz = self.get_mocked_resource()
@ -1320,3 +1326,80 @@ class LogicalDhcpServerTestCase(BaseTestResource):
def setUp(self):
super(LogicalDhcpServerTestCase, self).setUp(
resources.LogicalDhcpServer)
class DummyCachedResource(utils.NsxLibApiBase):
@property
def uri_segment(self):
return 'XXX'
@property
def resource_type(self):
return 'xxx'
@property
def use_cache_for_get(self):
return True
@property
def cache_timeout(self):
return 2
class ResourceCache(BaseTestResource):
def setUp(self):
super(ResourceCache, self).setUp(DummyCachedResource)
def test_get_with_cache(self):
mocked_resource = self.get_mocked_resource()
fake_uuid = uuidutils.generate_uuid()
# first call -> goes to the client
mocked_resource.get(fake_uuid)
self.assertEqual(1, test_client.mock_calls_count(
'get', mocked_resource))
# second call -> goes to cache
mocked_resource.get(fake_uuid)
self.assertEqual(1, test_client.mock_calls_count(
'get', mocked_resource))
# a different call -> goes to the client
fake_uuid2 = uuidutils.generate_uuid()
mocked_resource.get(fake_uuid2)
self.assertEqual(2, test_client.mock_calls_count(
'get', mocked_resource))
# third call -> still goes to cache
mocked_resource.get(fake_uuid)
self.assertEqual(2, test_client.mock_calls_count(
'get', mocked_resource))
# after timeout -> goes to the client
eventlet.sleep(2)
mocked_resource.get(fake_uuid)
self.assertEqual(3, test_client.mock_calls_count(
'get', mocked_resource))
# after delete -> goes to the client
mocked_resource.delete(fake_uuid)
mocked_resource.get(fake_uuid)
self.assertEqual(4, test_client.mock_calls_count(
'get', mocked_resource))
# And from cache again
mocked_resource.get(fake_uuid)
self.assertEqual(4, test_client.mock_calls_count(
'get', mocked_resource))
# Update the entry. The get inside the update is from
# the client too, because it must be current)
mocked_resource._update_with_retry(fake_uuid, {})
self.assertEqual(5, test_client.mock_calls_count(
'get', mocked_resource))
# after update -> goes to client
mocked_resource.get(fake_uuid)
self.assertEqual(6, test_client.mock_calls_count(
'get', mocked_resource))

View File

@ -378,8 +378,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
body = self._update_args(body, name, description)
if tags is not None:
body['tags'] = tags
return self._update_resource_with_retry(
self.get_path(profile_id), body)
return self._update_with_retry(profile_id, body)
def update_shaping(self, profile_id,
shaping_enabled=False,
@ -404,8 +403,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
else:
body = self._disable_shaping_in_args(body, direction=direction)
body = self._update_dscp_in_args(body, qos_marking, dscp)
return self._update_resource_with_retry(
self.get_path(profile_id), body)
return self._update_with_retry(profile_id, body)
def set_profile_shaping(self, profile_id,
ingress_bw_enabled=False,
@ -447,8 +445,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
body = self._update_dscp_in_args(body, qos_marking, dscp)
# update the profile in the backend
return self._update_resource_with_retry(
self.get_path(profile_id), body)
return self._update_with_retry(profile_id, body)
class NsxLibLogicalRouter(utils.NsxLibApiBase):
@ -689,6 +686,10 @@ class NsxLibTransportZone(utils.NsxLibApiBase):
def resource_type(self):
return 'TransportZone'
@property
def use_cache_for_get(self):
return True
def get_transport_type(self, uuid):
tz = self.get(uuid)
return tz['transport_type']
@ -719,6 +720,10 @@ class NsxLibDhcpRelayService(utils.NsxLibApiBase):
def resource_type(self):
return 'DhcpRelayService'
@property
def use_cache_for_get(self):
return True
def get_server_ips(self, uuid):
# Return the server ips of the relay profile attached to this service
service = self.get(uuid)
@ -737,6 +742,10 @@ class NsxLibDhcpRelayProfile(utils.NsxLibApiBase):
def resource_type(self):
return 'DhcpRelayProfile'
@property
def use_cache_for_get(self):
return True
def get_server_ips(self, uuid):
profile = self.get(uuid)
return profile.get('server_addresses')
@ -762,8 +771,7 @@ class NsxLibMetadataProxy(utils.NsxLibApiBase):
body['secret'] = secret
if edge_cluster_id is not None:
body['edge_cluster_id'] = edge_cluster_id
return self._update_resource_with_retry(
self.get_path(uuid), body)
return self._update_with_retry(uuid, body)
class NsxLibBridgeCluster(utils.NsxLibApiBase):

View File

@ -16,6 +16,7 @@
import abc
import inspect
import re
import time
from neutron_lib import exceptions
from oslo_log import log
@ -30,6 +31,7 @@ LOG = log.getLogger(__name__)
MAX_RESOURCE_TYPE_LEN = 20
MAX_TAG_LEN = 40
DEFAULT_MAX_ATTEMPTS = 10
DEFAULT_CACHE_AGE_SEC = 600
def _validate_resource_type_length(resource_type):
@ -181,6 +183,33 @@ def escape_tag_data(data):
return data.replace('/', '\\/').replace('-', '\\-')
class NsxLibCache(object):
def __init__(self, timeout):
self.timeout = timeout
self._cache = {}
super(NsxLibCache, self).__init__()
def expired(self, entry):
return (time.time() - entry['time']) > self.timeout
def get(self, key):
if key in self._cache:
# check that the value is still valid
if self.expired(self._cache[key]):
# this entry has expired
self.remove(key)
else:
return self._cache[key]['value']
def update(self, key, value):
self._cache[key] = {'time': time.time(),
'value': value}
def remove(self, key):
if key in self._cache:
del self._cache[key]
class NsxLibApiBase(object):
"""Base class for nsxlib api """
def __init__(self, client, nsxlib_config=None, nsxlib=None):
@ -188,6 +217,7 @@ class NsxLibApiBase(object):
self.nsxlib_config = nsxlib_config
self.nsxlib = nsxlib
super(NsxLibApiBase, self).__init__()
self.cache = NsxLibCache(self.cache_timeout)
@abc.abstractproperty
def uri_segment(self):
@ -197,6 +227,16 @@ class NsxLibApiBase(object):
def resource_type(self):
pass
@property
def use_cache_for_get(self):
"""By default no caching is used"""
return False
@property
def cache_timeout(self):
"""the default cache aging time in seconds"""
return DEFAULT_CACHE_AGE_SEC
def get_path(self, resource=None):
if resource:
return '%s/%s' % (self.uri_segment, resource)
@ -206,9 +246,23 @@ class NsxLibApiBase(object):
return self.client.list(self.uri_segment)
def get(self, uuid, silent=False):
return self.client.get(self.get_path(uuid), silent=silent)
if self.use_cache_for_get:
# try to get it from the cache
result = self.cache.get(uuid)
if result:
if not silent:
LOG.debug("Getting %s from cache.", self.get_path(uuid))
return result
# call the client
result = self.client.get(self.get_path(uuid), silent=silent)
if result and self.use_cache_for_get:
# add the result to the cache
self.cache.update(uuid, result)
return result
def delete(self, uuid):
if self.use_cache_for_get:
self.cache.remove(uuid)
return self.client.delete(self.get_path(uuid))
def find_by_display_name(self, display_name):
@ -218,6 +272,11 @@ class NsxLibApiBase(object):
found.append(resource)
return found
def _update_with_retry(self, uuid, payload):
if self.use_cache_for_get:
self.cache.remove(uuid)
return self._update_resource_with_retry(self.get_path(uuid), payload)
def _update_resource_with_retry(self, resource, payload):
# Using internal method so we can access max_attempts in the decorator
@retry_upon_exception(nsxlib_exceptions.StaleRevision,