Cache Get results for some nsxlib resources
Adding a caching mechanism to remember previous results of get commands and return them if they are not too old. This mechanism is disabled for most of the nsxlib resources, and used only by a few resources that are accessed frequently and modifies rarely such as transport zones. Change-Id: I4c1c723ee878feab9a86ff9015246c9e1773bd8b
This commit is contained in:
Adit Sarfaty
parent
2b43fa1a51
commit
8a5c545135
|
|
@ -252,6 +252,10 @@ class NsxClientTestCase(NsxLibTestCase):
|
|||
mock_call = getattr(self._record, verb.lower())
|
||||
mock_call.assert_any_call(**kwargs)
|
||||
|
||||
def call_count(self, verb):
|
||||
mock_call = getattr(self._record, verb.lower())
|
||||
return mock_call.call_count
|
||||
|
||||
@property
|
||||
def recorded_calls(self):
|
||||
return self._record
|
||||
|
|
|
|||
|
|
@ -67,6 +67,14 @@ def assert_call(verb, client_or_resource,
|
|||
'headers': headers, 'cert': None, 'timeout': timeout})
|
||||
|
||||
|
||||
def mock_calls_count(verb, client_or_resource):
|
||||
nsx_client = client_or_resource
|
||||
if getattr(nsx_client, 'client', None) is not None:
|
||||
nsx_client = nsx_client.client
|
||||
cluster = nsx_client._conn
|
||||
return cluster.call_count(verb)
|
||||
|
||||
|
||||
def assert_json_call(verb, client_or_resource, url,
|
||||
verify=nsxlib_testcase.NSX_CERT,
|
||||
data=None,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#
|
||||
import copy
|
||||
|
||||
import eventlet
|
||||
import mock
|
||||
|
||||
from oslo_serialization import jsonutils
|
||||
|
|
@ -28,6 +29,7 @@ from vmware_nsxlib.v3 import core_resources
|
|||
from vmware_nsxlib.v3 import exceptions
|
||||
from vmware_nsxlib.v3 import nsx_constants
|
||||
from vmware_nsxlib.v3 import resources
|
||||
from vmware_nsxlib.v3 import utils
|
||||
|
||||
|
||||
class BaseTestResource(nsxlib_testcase.NsxClientTestCase):
|
||||
|
|
@ -1118,6 +1120,10 @@ class TransportZone(BaseTestResource):
|
|||
tz_type = tz.get_transport_type(fake_tz['id'])
|
||||
self.assertEqual(tz.TRANSPORT_TYPE_OVERLAY, tz_type)
|
||||
|
||||
# call it again to test it when cached
|
||||
tz_type = tz.get_transport_type(fake_tz['id'])
|
||||
self.assertEqual(tz.TRANSPORT_TYPE_OVERLAY, tz_type)
|
||||
|
||||
def test_get_host_switch_mode(self):
|
||||
fake_tz = test_constants.FAKE_TZ.copy()
|
||||
tz = self.get_mocked_resource()
|
||||
|
|
@ -1320,3 +1326,80 @@ class LogicalDhcpServerTestCase(BaseTestResource):
|
|||
def setUp(self):
|
||||
super(LogicalDhcpServerTestCase, self).setUp(
|
||||
resources.LogicalDhcpServer)
|
||||
|
||||
|
||||
class DummyCachedResource(utils.NsxLibApiBase):
|
||||
|
||||
@property
|
||||
def uri_segment(self):
|
||||
return 'XXX'
|
||||
|
||||
@property
|
||||
def resource_type(self):
|
||||
return 'xxx'
|
||||
|
||||
@property
|
||||
def use_cache_for_get(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def cache_timeout(self):
|
||||
return 2
|
||||
|
||||
|
||||
class ResourceCache(BaseTestResource):
|
||||
|
||||
def setUp(self):
|
||||
super(ResourceCache, self).setUp(DummyCachedResource)
|
||||
|
||||
def test_get_with_cache(self):
|
||||
mocked_resource = self.get_mocked_resource()
|
||||
fake_uuid = uuidutils.generate_uuid()
|
||||
# first call -> goes to the client
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(1, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# second call -> goes to cache
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(1, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# a different call -> goes to the client
|
||||
fake_uuid2 = uuidutils.generate_uuid()
|
||||
mocked_resource.get(fake_uuid2)
|
||||
self.assertEqual(2, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# third call -> still goes to cache
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(2, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# after timeout -> goes to the client
|
||||
eventlet.sleep(2)
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(3, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# after delete -> goes to the client
|
||||
mocked_resource.delete(fake_uuid)
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(4, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# And from cache again
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(4, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# Update the entry. The get inside the update is from
|
||||
# the client too, because it must be current)
|
||||
mocked_resource._update_with_retry(fake_uuid, {})
|
||||
self.assertEqual(5, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
||||
# after update -> goes to client
|
||||
mocked_resource.get(fake_uuid)
|
||||
self.assertEqual(6, test_client.mock_calls_count(
|
||||
'get', mocked_resource))
|
||||
|
|
|
|||
|
|
@ -378,8 +378,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
|
|||
body = self._update_args(body, name, description)
|
||||
if tags is not None:
|
||||
body['tags'] = tags
|
||||
return self._update_resource_with_retry(
|
||||
self.get_path(profile_id), body)
|
||||
return self._update_with_retry(profile_id, body)
|
||||
|
||||
def update_shaping(self, profile_id,
|
||||
shaping_enabled=False,
|
||||
|
|
@ -404,8 +403,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
|
|||
else:
|
||||
body = self._disable_shaping_in_args(body, direction=direction)
|
||||
body = self._update_dscp_in_args(body, qos_marking, dscp)
|
||||
return self._update_resource_with_retry(
|
||||
self.get_path(profile_id), body)
|
||||
return self._update_with_retry(profile_id, body)
|
||||
|
||||
def set_profile_shaping(self, profile_id,
|
||||
ingress_bw_enabled=False,
|
||||
|
|
@ -447,8 +445,7 @@ class NsxLibQosSwitchingProfile(NsxLibSwitchingProfile):
|
|||
body = self._update_dscp_in_args(body, qos_marking, dscp)
|
||||
|
||||
# update the profile in the backend
|
||||
return self._update_resource_with_retry(
|
||||
self.get_path(profile_id), body)
|
||||
return self._update_with_retry(profile_id, body)
|
||||
|
||||
|
||||
class NsxLibLogicalRouter(utils.NsxLibApiBase):
|
||||
|
|
@ -689,6 +686,10 @@ class NsxLibTransportZone(utils.NsxLibApiBase):
|
|||
def resource_type(self):
|
||||
return 'TransportZone'
|
||||
|
||||
@property
|
||||
def use_cache_for_get(self):
|
||||
return True
|
||||
|
||||
def get_transport_type(self, uuid):
|
||||
tz = self.get(uuid)
|
||||
return tz['transport_type']
|
||||
|
|
@ -719,6 +720,10 @@ class NsxLibDhcpRelayService(utils.NsxLibApiBase):
|
|||
def resource_type(self):
|
||||
return 'DhcpRelayService'
|
||||
|
||||
@property
|
||||
def use_cache_for_get(self):
|
||||
return True
|
||||
|
||||
def get_server_ips(self, uuid):
|
||||
# Return the server ips of the relay profile attached to this service
|
||||
service = self.get(uuid)
|
||||
|
|
@ -737,6 +742,10 @@ class NsxLibDhcpRelayProfile(utils.NsxLibApiBase):
|
|||
def resource_type(self):
|
||||
return 'DhcpRelayProfile'
|
||||
|
||||
@property
|
||||
def use_cache_for_get(self):
|
||||
return True
|
||||
|
||||
def get_server_ips(self, uuid):
|
||||
profile = self.get(uuid)
|
||||
return profile.get('server_addresses')
|
||||
|
|
@ -762,8 +771,7 @@ class NsxLibMetadataProxy(utils.NsxLibApiBase):
|
|||
body['secret'] = secret
|
||||
if edge_cluster_id is not None:
|
||||
body['edge_cluster_id'] = edge_cluster_id
|
||||
return self._update_resource_with_retry(
|
||||
self.get_path(uuid), body)
|
||||
return self._update_with_retry(uuid, body)
|
||||
|
||||
|
||||
class NsxLibBridgeCluster(utils.NsxLibApiBase):
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
import abc
|
||||
import inspect
|
||||
import re
|
||||
import time
|
||||
|
||||
from neutron_lib import exceptions
|
||||
from oslo_log import log
|
||||
|
|
@ -30,6 +31,7 @@ LOG = log.getLogger(__name__)
|
|||
MAX_RESOURCE_TYPE_LEN = 20
|
||||
MAX_TAG_LEN = 40
|
||||
DEFAULT_MAX_ATTEMPTS = 10
|
||||
DEFAULT_CACHE_AGE_SEC = 600
|
||||
|
||||
|
||||
def _validate_resource_type_length(resource_type):
|
||||
|
|
@ -181,6 +183,33 @@ def escape_tag_data(data):
|
|||
return data.replace('/', '\\/').replace('-', '\\-')
|
||||
|
||||
|
||||
class NsxLibCache(object):
|
||||
def __init__(self, timeout):
|
||||
self.timeout = timeout
|
||||
self._cache = {}
|
||||
super(NsxLibCache, self).__init__()
|
||||
|
||||
def expired(self, entry):
|
||||
return (time.time() - entry['time']) > self.timeout
|
||||
|
||||
def get(self, key):
|
||||
if key in self._cache:
|
||||
# check that the value is still valid
|
||||
if self.expired(self._cache[key]):
|
||||
# this entry has expired
|
||||
self.remove(key)
|
||||
else:
|
||||
return self._cache[key]['value']
|
||||
|
||||
def update(self, key, value):
|
||||
self._cache[key] = {'time': time.time(),
|
||||
'value': value}
|
||||
|
||||
def remove(self, key):
|
||||
if key in self._cache:
|
||||
del self._cache[key]
|
||||
|
||||
|
||||
class NsxLibApiBase(object):
|
||||
"""Base class for nsxlib api """
|
||||
def __init__(self, client, nsxlib_config=None, nsxlib=None):
|
||||
|
|
@ -188,6 +217,7 @@ class NsxLibApiBase(object):
|
|||
self.nsxlib_config = nsxlib_config
|
||||
self.nsxlib = nsxlib
|
||||
super(NsxLibApiBase, self).__init__()
|
||||
self.cache = NsxLibCache(self.cache_timeout)
|
||||
|
||||
@abc.abstractproperty
|
||||
def uri_segment(self):
|
||||
|
|
@ -197,6 +227,16 @@ class NsxLibApiBase(object):
|
|||
def resource_type(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def use_cache_for_get(self):
|
||||
"""By default no caching is used"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def cache_timeout(self):
|
||||
"""the default cache aging time in seconds"""
|
||||
return DEFAULT_CACHE_AGE_SEC
|
||||
|
||||
def get_path(self, resource=None):
|
||||
if resource:
|
||||
return '%s/%s' % (self.uri_segment, resource)
|
||||
|
|
@ -206,9 +246,23 @@ class NsxLibApiBase(object):
|
|||
return self.client.list(self.uri_segment)
|
||||
|
||||
def get(self, uuid, silent=False):
|
||||
return self.client.get(self.get_path(uuid), silent=silent)
|
||||
if self.use_cache_for_get:
|
||||
# try to get it from the cache
|
||||
result = self.cache.get(uuid)
|
||||
if result:
|
||||
if not silent:
|
||||
LOG.debug("Getting %s from cache.", self.get_path(uuid))
|
||||
return result
|
||||
# call the client
|
||||
result = self.client.get(self.get_path(uuid), silent=silent)
|
||||
if result and self.use_cache_for_get:
|
||||
# add the result to the cache
|
||||
self.cache.update(uuid, result)
|
||||
return result
|
||||
|
||||
def delete(self, uuid):
|
||||
if self.use_cache_for_get:
|
||||
self.cache.remove(uuid)
|
||||
return self.client.delete(self.get_path(uuid))
|
||||
|
||||
def find_by_display_name(self, display_name):
|
||||
|
|
@ -218,6 +272,11 @@ class NsxLibApiBase(object):
|
|||
found.append(resource)
|
||||
return found
|
||||
|
||||
def _update_with_retry(self, uuid, payload):
|
||||
if self.use_cache_for_get:
|
||||
self.cache.remove(uuid)
|
||||
return self._update_resource_with_retry(self.get_path(uuid), payload)
|
||||
|
||||
def _update_resource_with_retry(self, resource, payload):
|
||||
# Using internal method so we can access max_attempts in the decorator
|
||||
@retry_upon_exception(nsxlib_exceptions.StaleRevision,
|
||||
|
|
|
|||
Loading…
Reference in New Issue