From 9227352b39e65c8d457e91a69921d52139d048a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Wed, 11 Mar 2020 16:23:30 +0100 Subject: [PATCH] Adding functional tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This project lack of functional tests, these changes aim to introduce them. These changes activate functional tests for oslo.cache. They introduced: - related ci jobs - splits tests into functional and unit - bindep to manage backends server Functional tests support the following backends for now: - etcd3gw Co-authored-by: Radosław Piliszek Change-Id: I604cb542ac25fd4bfac57ea607b447ac82404b7e --- .gitignore | 2 + .stestr.conf | 3 +- .zuul.yaml | 22 +- bindep.txt | 4 + ...ckend_etcd3gw.py => test_cache_backend.py} | 73 ++-- oslo_cache/tests/test_cache.py | 322 ----------------- oslo_cache/tests/unit/__init__.py | 0 .../{ => unit}/test_cache_backend_mongo.py | 0 oslo_cache/tests/unit/test_cache_basics.py | 338 ++++++++++++++++++ .../tests/{ => unit}/test_connection_pool.py | 0 .../tests/{ => unit}/test_dict_backend.py | 0 playbooks/tests/functional/pre.yml | 17 + tox.ini | 8 +- 13 files changed, 424 insertions(+), 365 deletions(-) create mode 100644 bindep.txt rename oslo_cache/tests/functional/{test_cache_backend_etcd3gw.py => test_cache_backend.py} (85%) create mode 100644 oslo_cache/tests/unit/__init__.py rename oslo_cache/tests/{ => unit}/test_cache_backend_mongo.py (100%) create mode 100644 oslo_cache/tests/unit/test_cache_basics.py rename oslo_cache/tests/{ => unit}/test_connection_pool.py (100%) rename oslo_cache/tests/{ => unit}/test_dict_backend.py (100%) create mode 100644 playbooks/tests/functional/pre.yml diff --git a/.gitignore b/.gitignore index cec106ea..e85e2abb 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ releasenotes/build RELEASENOTES.rst releasenotes/notes/reno.cache +# pifpaf and testing artifacts +etcd-v* diff --git a/.stestr.conf b/.stestr.conf index f2027908..80890208 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,3 +1,2 @@ [DEFAULT] -test_path=./oslo_cache/tests -top_path=./ +test_path=${STESTR_TEST_PATH:-./oslo_cache/tests/unit} diff --git a/.zuul.yaml b/.zuul.yaml index 8fbdaabf..f7ba1d78 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,12 +1,18 @@ - job: - name: oslo.cache-tempest-full-py3-memcached-pool - parent: tempest-full-py3 - voting: false - description: | - Base integration test with memcache_pool backend. + name: oslo.cache-functional-tox-py38 + parent: openstack-tox-py38 + abstract: true + pre-run: playbooks/tests/functional/pre.yml vars: - devstack_localrc: - CACHE_BACKEND: oslo_cache.memcache_pool + tox_envlist: functional + +- job: + name: oslo.cache-functional-tox-py38-etcd3gw + parent: oslo.cache-functional-tox-py38 + vars: + tox_environment: + OSLO_CACHE_BACKEND: etcd3gw + OSLO_CACHE_BACKEND_DAEMON: etcd - project: templates: @@ -19,4 +25,4 @@ - release-notes-jobs-python3 check: jobs: - - oslo.cache-tempest-full-py3-memcached-pool + - oslo.cache-functional-tox-py38-etcd3gw diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 00000000..d374994a --- /dev/null +++ b/bindep.txt @@ -0,0 +1,4 @@ +# This file contains runtime (non-python) dependencies +# More info at: https://docs.openstack.org/infra/bindep/readme.html + +etcd [tests-functional-etcd] diff --git a/oslo_cache/tests/functional/test_cache_backend_etcd3gw.py b/oslo_cache/tests/functional/test_cache_backend.py similarity index 85% rename from oslo_cache/tests/functional/test_cache_backend_etcd3gw.py rename to oslo_cache/tests/functional/test_cache_backend.py index d8fe5440..edc19919 100644 --- a/oslo_cache/tests/functional/test_cache_backend_etcd3gw.py +++ b/oslo_cache/tests/functional/test_cache_backend.py @@ -12,38 +12,51 @@ # License for the specific language governing permissions and limitations # under the License. +import os + from dogpile.cache import region as dp_region from oslo_utils import uuidutils -import urllib3 -from oslo_cache import core -from oslo_cache.tests import test_cache +from oslotest import base + +from oslo_cache import core as cache -NO_VALUE = core.NO_VALUE +NO_VALUE = cache.NO_VALUE + +TESTABLE_BACKENDS = [ + 'etcd3gw', +] -class Etcd3gwCache(test_cache.BaseTestCase): - arguments = { - 'host': '127.0.0.1', - 'port': 2379, - } +class TestCacheBackend(base.BaseTestCase): + arguments = {} def setUp(self): - test_cache.BaseTestCase.setUp(self) - try: - urllib3.PoolManager().request( - 'GET', - '%s:%d' % (self.arguments['host'], self.arguments['port']) - ) - return True - except urllib3.exceptions.HTTPError: - self.skipTest("skipping this test") + super(TestCacheBackend, self).setUp() + self.backend = os.getenv('OSLO_CACHE_BACKEND') + if self.backend not in TESTABLE_BACKENDS: + raise Exception( + "Backend (%s) not supported in tests" % self.backend) + if self.backend == 'etcd3gw': + self.backend = "oslo_cache.etcd3gw" + # TODO(hberaud): Ideally functional tests should be similar + # to the usage examples given in the oslo.cache documentation [1]. + # Config [2] should be used to pass arguments to the backend + # rather than using direct usage like below. + # It could help us to made the tests interoperable between + # the different backends in use in tests. + # [1] https://docs.openstack.org/oslo.cache/latest/user/usage.html + # [2] https://docs.openstack.org/oslo.cache/latest/configuration/index.html # noqa + self.arguments = { + 'host': '127.0.0.1', + 'port': 2379, + } def test_typical_configuration(self): dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) self.assertTrue(True) # reached here means no initialization error @@ -51,7 +64,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_get_missing_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -61,7 +74,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_set_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -72,7 +85,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_set_none_as_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -83,7 +96,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_set_blank_as_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -94,7 +107,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_set_same_key_multiple_times(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -112,7 +125,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_multi_set_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) random_key = uuidutils.generate_uuid(dashed=False) @@ -133,7 +146,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_multi_get_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) random_key = uuidutils.generate_uuid(dashed=False) @@ -156,7 +169,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_multi_set_should_update_existing(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) random_key = uuidutils.generate_uuid(dashed=False) @@ -184,7 +197,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_multi_set_get_with_blanks_none(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) random_key = uuidutils.generate_uuid(dashed=False) @@ -225,7 +238,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_delete_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) @@ -240,7 +253,7 @@ class Etcd3gwCache(test_cache.BaseTestCase): def test_backend_multi_delete_data(self): region = dp_region.make_region().configure( - 'oslo_cache.etcd3gw', + self.backend, arguments=self.arguments ) random_key = uuidutils.generate_uuid(dashed=False) diff --git a/oslo_cache/tests/test_cache.py b/oslo_cache/tests/test_cache.py index 353cc64d..6042dcd3 100644 --- a/oslo_cache/tests/test_cache.py +++ b/oslo_cache/tests/test_cache.py @@ -13,25 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. -import copy -import time -from unittest import mock - -from dogpile.cache import proxy -from oslo_config import cfg from oslo_config import fixture as config_fixture -from oslo_utils import uuidutils from oslotest import base -from oslo_cache import _opts -from oslo_cache import core as cache -from oslo_cache import exception - - -NO_VALUE = cache.NO_VALUE -TEST_GROUP = uuidutils.generate_uuid(dashed=False) -TEST_GROUP2 = uuidutils.generate_uuid(dashed=False) - class BaseTestCase(base.BaseTestCase): def setUp(self): @@ -44,309 +28,3 @@ class BaseTestCase(base.BaseTestCase): backend='dogpile.cache.memory', enabled=True, proxies=['oslo_cache.testing.CacheIsolatingProxy']) - - -def _copy_value(value): - if value is not NO_VALUE: - value = copy.deepcopy(value) - return value - - -class TestProxy(proxy.ProxyBackend): - def get(self, key): - value = _copy_value(self.proxied.get(key)) - if value is not NO_VALUE: - if isinstance(value[0], TestProxyValue): - value[0].cached = True - return value - - -class TestProxyValue(object): - def __init__(self, value): - self.value = value - self.cached = False - - -class CacheRegionTest(BaseTestCase): - - def setUp(self): - super(CacheRegionTest, self).setUp() - self.region = cache.create_region() - cache.configure_cache_region(self.config_fixture.conf, self.region) - self.region.wrap(TestProxy) - self.region_kwargs = cache.create_region( - function=cache.kwarg_function_key_generator) - cache.configure_cache_region(self.config_fixture.conf, - self.region_kwargs) - self.region_kwargs.wrap(TestProxy) - self.test_value = TestProxyValue('Decorator Test') - - def _add_test_caching_option(self): - self.config_fixture.register_opt( - cfg.BoolOpt('caching', default=True), group='cache') - - def _add_dummy_config_group(self): - self.config_fixture.register_opt( - cfg.IntOpt('cache_time'), group=TEST_GROUP) - self.config_fixture.register_opt( - cfg.IntOpt('cache_time'), group=TEST_GROUP2) - - def _get_cacheable_function(self, region=None): - region = region if region else self.region - memoize = cache.get_memoization_decorator( - self.config_fixture.conf, region, group='cache') - - @memoize - def cacheable_function(value=0, **kw): - return value - - return cacheable_function - - def test_region_built_with_proxy_direct_cache_test(self): - # Verify cache regions are properly built with proxies. - test_value = TestProxyValue('Direct Cache Test') - self.region.set('cache_test', test_value) - cached_value = self.region.get('cache_test') - self.assertTrue(cached_value.cached) - - def test_cache_region_no_error_multiple_config(self): - # Verify configuring the CacheRegion again doesn't error. - cache.configure_cache_region(self.config_fixture.conf, self.region) - cache.configure_cache_region(self.config_fixture.conf, self.region) - - def _get_cache_fallthrough_fn(self, cache_time): - memoize = cache.get_memoization_decorator( - self.config_fixture.conf, - self.region, - group='cache', - expiration_group=TEST_GROUP2) - - class _test_obj(object): - def __init__(self, value): - self.test_value = value - - @memoize - def get_test_value(self): - return self.test_value - - def _do_test(value): - - test_obj = _test_obj(value) - - # Ensure the value has been cached - test_obj.get_test_value() - # Get the now cached value - cached_value = test_obj.get_test_value() - self.assertTrue(cached_value.cached) - self.assertEqual(value.value, cached_value.value) - self.assertEqual(cached_value.value, test_obj.test_value.value) - # Change the underlying value on the test object. - test_obj.test_value = TestProxyValue( - uuidutils.generate_uuid(dashed=False)) - self.assertEqual(cached_value.value, - test_obj.get_test_value().value) - # override the system time to ensure the non-cached new value - # is returned - new_time = time.time() + (cache_time * 2) - with mock.patch.object(time, 'time', - return_value=new_time): - overriden_cache_value = test_obj.get_test_value() - self.assertNotEqual(cached_value.value, - overriden_cache_value.value) - self.assertEqual(test_obj.test_value.value, - overriden_cache_value.value) - - return _do_test - - def test_cache_no_fallthrough_expiration_time_fn(self): - self._add_dummy_config_group() - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] group - cache_time = 600 - expiration_time = cache._get_expiration_time_fn( - self.config_fixture.conf, TEST_GROUP) - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the dummy group cache_time value - self.config_fixture.config(cache_time=cache_time, - group=TEST_GROUP) - test_value = TestProxyValue(uuidutils.generate_uuid(dashed=False)) - self.assertEqual(cache_time, expiration_time()) - do_test(value=test_value) - - def test_cache_fallthrough_expiration_time_fn(self): - self._add_dummy_config_group() - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] group - cache_time = 599 - expiration_time = cache._get_expiration_time_fn( - self.config_fixture.conf, TEST_GROUP) - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the dummy group cache_time value set to None and - # the global value set. - self.config_fixture.config(cache_time=None, group=TEST_GROUP) - test_value = TestProxyValue( - uuidutils.generate_uuid(dashed=False)) - self.assertIsNone(expiration_time()) - do_test(value=test_value) - - def test_should_cache_fn_global_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=True) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=False) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled_group_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled and the specific - # group caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=False) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_group_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # group caching disabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=False) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_group_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # group caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_cache_dictionary_config_builder(self): - """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='oslo_cache.dict', - expiration_time=86400, - backend_argument=['arg1:test', - 'arg2:test:test', - 'arg3.invalid']) - - config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertEqual( - self.config_fixture.conf.cache.backend, - config_dict['test_prefix.backend']) - self.assertEqual( - self.config_fixture.conf.cache.expiration_time, - config_dict['test_prefix.expiration_time']) - self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) - self.assertEqual('test:test', - config_dict['test_prefix.arguments.arg2']) - self.assertNotIn('test_prefix.arguments.arg3', config_dict) - - def test_cache_dictionary_config_builder_global_disabled(self): - """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=False, - config_prefix='test_prefix', - backend='oslo_cache.dict') - - self.assertFalse(self.config_fixture.conf.cache.enabled) - config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertEqual( - _opts._DEFAULT_BACKEND, - config_dict['test_prefix.backend']) - - def test_cache_debug_proxy(self): - single_value = 'Test Value' - single_key = 'testkey' - multi_values = {'key1': 1, 'key2': 2, 'key3': 3} - - self.region.set(single_key, single_value) - self.assertEqual(single_value, self.region.get(single_key)) - - self.region.delete(single_key) - self.assertEqual(NO_VALUE, self.region.get(single_key)) - - self.region.set_multi(multi_values) - cached_values = self.region.get_multi(multi_values.keys()) - for value in multi_values.values(): - self.assertIn(value, cached_values) - self.assertEqual(len(multi_values.values()), len(cached_values)) - - self.region.delete_multi(multi_values.keys()) - for value in self.region.get_multi(multi_values.keys()): - self.assertEqual(NO_VALUE, value) - - def test_configure_non_region_object_raises_error(self): - self.assertRaises(exception.ConfigurationError, - cache.configure_cache_region, - self.config_fixture.conf, - "bogus") - - def test_kwarg_function_key_generator_no_kwargs(self): - cacheable_function = self._get_cacheable_function( - region=self.region_kwargs) - - self.config_fixture.config(group='cache', enabled=True) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_kwarg_function_key_generator_with_kwargs(self): - cacheable_function = self._get_cacheable_function( - region=self.region_kwargs) - - self.config_fixture.config(group='cache', enabled=True) - cacheable_function(value=self.test_value) - cached_value = cacheable_function(value=self.test_value) - self.assertTrue(cached_value.cached) - - -class UTF8KeyManglerTests(BaseTestCase): - - def test_key_is_utf8_encoded(self): - key = u'fäké1' - encoded = cache._sha1_mangle_key(key) - self.assertIsNotNone(encoded) - - def test_key_is_bytestring(self): - key = b'\xcf\x84o\xcf\x81\xce\xbdo\xcf\x82' - encoded = cache._sha1_mangle_key(key) - self.assertIsNotNone(encoded) - - def test_key_is_string(self): - key = 'fake' - encoded = cache._sha1_mangle_key(key) - self.assertIsNotNone(encoded) diff --git a/oslo_cache/tests/unit/__init__.py b/oslo_cache/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_cache/tests/test_cache_backend_mongo.py b/oslo_cache/tests/unit/test_cache_backend_mongo.py similarity index 100% rename from oslo_cache/tests/test_cache_backend_mongo.py rename to oslo_cache/tests/unit/test_cache_backend_mongo.py diff --git a/oslo_cache/tests/unit/test_cache_basics.py b/oslo_cache/tests/unit/test_cache_basics.py new file mode 100644 index 00000000..781a4389 --- /dev/null +++ b/oslo_cache/tests/unit/test_cache_basics.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +# Copyright 2013 Metacloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import time +from unittest import mock + +from dogpile.cache import proxy +from oslo_config import cfg +from oslo_utils import uuidutils + +from oslo_cache import _opts +from oslo_cache import core as cache +from oslo_cache import exception +from oslo_cache.tests import test_cache + + +NO_VALUE = cache.NO_VALUE +TEST_GROUP = uuidutils.generate_uuid(dashed=False) +TEST_GROUP2 = uuidutils.generate_uuid(dashed=False) + + +def _copy_value(value): + if value is not NO_VALUE: + value = copy.deepcopy(value) + return value + + +class TestProxy(proxy.ProxyBackend): + def get(self, key): + value = _copy_value(self.proxied.get(key)) + if value is not NO_VALUE: + if isinstance(value[0], TestProxyValue): + value[0].cached = True + return value + + +class TestProxyValue(object): + def __init__(self, value): + self.value = value + self.cached = False + + +class CacheRegionTest(test_cache.BaseTestCase): + + def setUp(self): + super(CacheRegionTest, self).setUp() + self.region = cache.create_region() + cache.configure_cache_region(self.config_fixture.conf, self.region) + self.region.wrap(TestProxy) + self.region_kwargs = cache.create_region( + function=cache.kwarg_function_key_generator) + cache.configure_cache_region(self.config_fixture.conf, + self.region_kwargs) + self.region_kwargs.wrap(TestProxy) + self.test_value = TestProxyValue('Decorator Test') + + def _add_test_caching_option(self): + self.config_fixture.register_opt( + cfg.BoolOpt('caching', default=True), group='cache') + + def _add_dummy_config_group(self): + self.config_fixture.register_opt( + cfg.IntOpt('cache_time'), group=TEST_GROUP) + self.config_fixture.register_opt( + cfg.IntOpt('cache_time'), group=TEST_GROUP2) + + def _get_cacheable_function(self, region=None): + region = region if region else self.region + memoize = cache.get_memoization_decorator( + self.config_fixture.conf, region, group='cache') + + @memoize + def cacheable_function(value=0, **kw): + return value + + return cacheable_function + + def test_region_built_with_proxy_direct_cache_test(self): + # Verify cache regions are properly built with proxies. + test_value = TestProxyValue('Direct Cache Test') + self.region.set('cache_test', test_value) + cached_value = self.region.get('cache_test') + self.assertTrue(cached_value.cached) + + def test_cache_region_no_error_multiple_config(self): + # Verify configuring the CacheRegion again doesn't error. + cache.configure_cache_region(self.config_fixture.conf, self.region) + cache.configure_cache_region(self.config_fixture.conf, self.region) + + def _get_cache_fallthrough_fn(self, cache_time): + memoize = cache.get_memoization_decorator( + self.config_fixture.conf, + self.region, + group='cache', + expiration_group=TEST_GROUP2) + + class _test_obj(object): + def __init__(self, value): + self.test_value = value + + @memoize + def get_test_value(self): + return self.test_value + + def _do_test(value): + + test_obj = _test_obj(value) + + # Ensure the value has been cached + test_obj.get_test_value() + # Get the now cached value + cached_value = test_obj.get_test_value() + self.assertTrue(cached_value.cached) + self.assertEqual(value.value, cached_value.value) + self.assertEqual(cached_value.value, test_obj.test_value.value) + # Change the underlying value on the test object. + test_obj.test_value = TestProxyValue( + uuidutils.generate_uuid(dashed=False)) + self.assertEqual(cached_value.value, + test_obj.get_test_value().value) + # override the system time to ensure the non-cached new value + # is returned + new_time = time.time() + (cache_time * 2) + with mock.patch.object(time, 'time', + return_value=new_time): + overriden_cache_value = test_obj.get_test_value() + self.assertNotEqual(cached_value.value, + overriden_cache_value.value) + self.assertEqual(test_obj.test_value.value, + overriden_cache_value.value) + + return _do_test + + def test_cache_no_fallthrough_expiration_time_fn(self): + self._add_dummy_config_group() + # Since we do not re-configure the cache region, for ease of testing + # this value is set the same as the expiration_time default in the + # [cache] group + cache_time = 600 + expiration_time = cache._get_expiration_time_fn( + self.config_fixture.conf, TEST_GROUP) + do_test = self._get_cache_fallthrough_fn(cache_time) + # Run the test with the dummy group cache_time value + self.config_fixture.config(cache_time=cache_time, + group=TEST_GROUP) + test_value = TestProxyValue(uuidutils.generate_uuid(dashed=False)) + self.assertEqual(cache_time, expiration_time()) + do_test(value=test_value) + + def test_cache_fallthrough_expiration_time_fn(self): + self._add_dummy_config_group() + # Since we do not re-configure the cache region, for ease of testing + # this value is set the same as the expiration_time default in the + # [cache] group + cache_time = 599 + expiration_time = cache._get_expiration_time_fn( + self.config_fixture.conf, TEST_GROUP) + do_test = self._get_cache_fallthrough_fn(cache_time) + # Run the test with the dummy group cache_time value set to None and + # the global value set. + self.config_fixture.config(cache_time=None, group=TEST_GROUP) + test_value = TestProxyValue( + uuidutils.generate_uuid(dashed=False)) + self.assertIsNone(expiration_time()) + do_test(value=test_value) + + def test_should_cache_fn_global_cache_enabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally enabled. + cacheable_function = self._get_cacheable_function() + + self.config_fixture.config(group='cache', enabled=True) + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertTrue(cached_value.cached) + + def test_should_cache_fn_global_cache_disabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally disabled. + cacheable_function = self._get_cacheable_function() + + self.config_fixture.config(group='cache', enabled=False) + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + def test_should_cache_fn_global_cache_disabled_group_cache_enabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally disabled and the specific + # group caching enabled. + cacheable_function = self._get_cacheable_function() + + self._add_test_caching_option() + self.config_fixture.config(group='cache', enabled=False) + self.config_fixture.config(group='cache', caching=True) + + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + def test_should_cache_fn_global_cache_enabled_group_cache_disabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally enabled and the specific + # group caching disabled. + cacheable_function = self._get_cacheable_function() + + self._add_test_caching_option() + self.config_fixture.config(group='cache', enabled=True) + self.config_fixture.config(group='cache', caching=False) + + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + def test_should_cache_fn_global_cache_enabled_group_cache_enabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally enabled and the specific + # group caching enabled. + cacheable_function = self._get_cacheable_function() + + self._add_test_caching_option() + self.config_fixture.config(group='cache', enabled=True) + self.config_fixture.config(group='cache', caching=True) + + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertTrue(cached_value.cached) + + def test_cache_dictionary_config_builder(self): + """Validate we build a sane dogpile.cache dictionary config.""" + self.config_fixture.config(group='cache', + config_prefix='test_prefix', + backend='oslo_cache.dict', + expiration_time=86400, + backend_argument=['arg1:test', + 'arg2:test:test', + 'arg3.invalid']) + + config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual( + self.config_fixture.conf.cache.backend, + config_dict['test_prefix.backend']) + self.assertEqual( + self.config_fixture.conf.cache.expiration_time, + config_dict['test_prefix.expiration_time']) + self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) + self.assertEqual('test:test', + config_dict['test_prefix.arguments.arg2']) + self.assertNotIn('test_prefix.arguments.arg3', config_dict) + + def test_cache_dictionary_config_builder_global_disabled(self): + """Validate the backend is reset to default if caching is disabled.""" + self.config_fixture.config(group='cache', + enabled=False, + config_prefix='test_prefix', + backend='oslo_cache.dict') + + self.assertFalse(self.config_fixture.conf.cache.enabled) + config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual( + _opts._DEFAULT_BACKEND, + config_dict['test_prefix.backend']) + + def test_cache_debug_proxy(self): + single_value = 'Test Value' + single_key = 'testkey' + multi_values = {'key1': 1, 'key2': 2, 'key3': 3} + + self.region.set(single_key, single_value) + self.assertEqual(single_value, self.region.get(single_key)) + + self.region.delete(single_key) + self.assertEqual(NO_VALUE, self.region.get(single_key)) + + self.region.set_multi(multi_values) + cached_values = self.region.get_multi(multi_values.keys()) + for value in multi_values.values(): + self.assertIn(value, cached_values) + self.assertEqual(len(multi_values.values()), len(cached_values)) + + self.region.delete_multi(multi_values.keys()) + for value in self.region.get_multi(multi_values.keys()): + self.assertEqual(NO_VALUE, value) + + def test_configure_non_region_object_raises_error(self): + self.assertRaises(exception.ConfigurationError, + cache.configure_cache_region, + self.config_fixture.conf, + "bogus") + + def test_kwarg_function_key_generator_no_kwargs(self): + cacheable_function = self._get_cacheable_function( + region=self.region_kwargs) + + self.config_fixture.config(group='cache', enabled=True) + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertTrue(cached_value.cached) + + def test_kwarg_function_key_generator_with_kwargs(self): + cacheable_function = self._get_cacheable_function( + region=self.region_kwargs) + + self.config_fixture.config(group='cache', enabled=True) + cacheable_function(value=self.test_value) + cached_value = cacheable_function(value=self.test_value) + self.assertTrue(cached_value.cached) + + +class UTF8KeyManglerTests(test_cache.BaseTestCase): + + def test_key_is_utf8_encoded(self): + key = u'fäké1' + encoded = cache._sha1_mangle_key(key) + self.assertIsNotNone(encoded) + + def test_key_is_bytestring(self): + key = b'\xcf\x84o\xcf\x81\xce\xbdo\xcf\x82' + encoded = cache._sha1_mangle_key(key) + self.assertIsNotNone(encoded) + + def test_key_is_string(self): + key = 'fake' + encoded = cache._sha1_mangle_key(key) + self.assertIsNotNone(encoded) diff --git a/oslo_cache/tests/test_connection_pool.py b/oslo_cache/tests/unit/test_connection_pool.py similarity index 100% rename from oslo_cache/tests/test_connection_pool.py rename to oslo_cache/tests/unit/test_connection_pool.py diff --git a/oslo_cache/tests/test_dict_backend.py b/oslo_cache/tests/unit/test_dict_backend.py similarity index 100% rename from oslo_cache/tests/test_dict_backend.py rename to oslo_cache/tests/unit/test_dict_backend.py diff --git a/playbooks/tests/functional/pre.yml b/playbooks/tests/functional/pre.yml new file mode 100644 index 00000000..3ec145b1 --- /dev/null +++ b/playbooks/tests/functional/pre.yml @@ -0,0 +1,17 @@ +- hosts: all + vars: + oslo_cache_backend_daemon: "{{ tox_environment.OSLO_CACHE_BACKEND_DAEMON | default(tox_environment.OSLO_CACHE_BACKEND) }}" + roles: + - role: bindep + bindep_profile: "tests-functional-{{ oslo_cache_backend_daemon }}" + tasks: + # NOTE(yoctozepto): Debian and Ubuntu have this nasty policy of starting + # installed services for us. We don't rely on system-wide service and use + # pifpaf. Unfortunately, default port may conflict with system-wide service. + # So, for sanity and resource conservation, let's stop it before tests run. + - name: "Stop {{ oslo_cache_backend_daemon }}" + service: + name: "{{ oslo_cache_backend_daemon }}" + state: stopped + enabled: no + become: yes diff --git a/tox.ini b/tox.ini index 349a43a5..7d769ab3 100644 --- a/tox.ini +++ b/tox.ini @@ -14,11 +14,13 @@ commands = find . -type f -name "*.pyc" -delete stestr run --slowest {posargs} -[testenv:py35-functional-etcd3gw] -basepython = python3.5 +[testenv:functional] +setenv = + STESTR_TEST_PATH=./oslo_cache/tests/functional +passenv = OSLO_CACHE_BACKEND commands = find . -type f -name "*.pyc" -delete - {toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd -- stestr run --slowest functional.* + pifpaf -e OSLO_CACHE_TEST run {env:OSLO_CACHE_BACKEND_DAEMON:{env:OSLO_CACHE_BACKEND}} -- stestr run --slowest [testenv:pep8] deps = {[testenv]deps}