diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56cb167a..b4079be4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,13 +18,9 @@ repos: hooks: - id: hacking additional_dependencies: [] - - repo: https://github.com/PyCQA/bandit - rev: 1.8.6 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.0 hooks: - - id: bandit - args: ['-x', 'tests'] - - repo: https://github.com/asottile/pyupgrade - rev: v3.20.0 - hooks: - - id: pyupgrade - args: [--py310-plus] + - id: ruff-check + args: ['--fix', '--unsafe-fixes'] + - id: ruff-format diff --git a/doc/source/conf.py b/doc/source/conf.py index 3a763080..1cdb545a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -34,9 +34,7 @@ openstackdocs_bug_tag = '' # sphinxcontrib.apidoc options apidoc_module_dir = '../../oslo_cache' apidoc_output_dir = 'reference/api' -apidoc_excluded_paths = [ - 'tests' -] +apidoc_excluded_paths = ['tests'] # The suffix of source filenames. source_suffix = '.rst' @@ -70,16 +68,19 @@ html_theme = 'openstackdocs' # html_static_path = ['static'] # Output file base name for HTML help builder. -htmlhelp_basename = '%sdoc' % project +htmlhelp_basename = f'{project}doc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass # [howto/manual]). latex_documents = [ - ('index', - '%s.tex' % project, - '%s Documentation' % project, - 'OpenStack Foundation', 'manual'), + ( + 'index', + f'{project}.tex', + f'{project} Documentation', + 'OpenStack Foundation', + 'manual', + ), ] intersphinx_mapping = { diff --git a/oslo_cache/__init__.py b/oslo_cache/__init__.py index 6dd171ce..977202f5 100644 --- a/oslo_cache/__init__.py +++ b/oslo_cache/__init__.py @@ -11,8 +11,13 @@ # under the License. -from oslo_cache.core import * # noqa - +from oslo_cache.core import ( + configure, + configure_cache_region, + create_region, + get_memoization_decorator, + NO_VALUE, +) __all__ = [ 'configure', diff --git a/oslo_cache/_bmemcache_pool.py b/oslo_cache/_bmemcache_pool.py index 2a172d3c..eee22eec 100644 --- a/oslo_cache/_bmemcache_pool.py +++ b/oslo_cache/_bmemcache_pool.py @@ -14,7 +14,9 @@ # under the License. """Thread-safe connection pool for python-binary-memcached.""" + import debtcollector + try: import eventlet except ImportError: @@ -28,7 +30,8 @@ LOG = log.getLogger(__name__) if eventlet and eventlet.patcher.is_monkey_patched('thread'): debtcollector.deprecate( - "Eventlet support is deprecated and will be removed.") + "Eventlet support is deprecated and will be removed." + ) class _BMemcacheClient(bmemcached.Client): @@ -38,6 +41,7 @@ class _BMemcacheClient(bmemcached.Client): methods overloaded by threading.local so we can reuse clients in different threads """ + __delattr__ = object.__delattr__ __getattribute__ = object.__getattribute__ __setattr__ = object.__setattr__ diff --git a/oslo_cache/_memcache_pool.py b/oslo_cache/_memcache_pool.py index 578dba47..3599fc2e 100644 --- a/oslo_cache/_memcache_pool.py +++ b/oslo_cache/_memcache_pool.py @@ -23,6 +23,7 @@ import threading import time import debtcollector + try: import eventlet except ImportError: @@ -39,7 +40,8 @@ LOG = log.getLogger(__name__) if eventlet and eventlet.patcher.is_monkey_patched('thread'): debtcollector.deprecate( - "Eventlet support is deprecated and will be removed.") + "Eventlet support is deprecated and will be removed." + ) class _MemcacheClient(memcache.Client): @@ -49,6 +51,7 @@ class _MemcacheClient(memcache.Client): methods overloaded by threading.local so we can reuse clients in different threads """ + __delattr__ = object.__delattr__ __getattribute__ = object.__getattribute__ __setattr__ = object.__setattr__ @@ -75,6 +78,7 @@ class ConnectionPool(queue.Queue): This class implements the basic connection pool logic as an abstract base class. """ + def __init__(self, maxsize, unused_timeout, conn_get_timeout=None): """Initialize the connection pool. @@ -118,7 +122,8 @@ class ConnectionPool(queue.Queue): break except Exception as e: self._do_log( - LOG.warning, "Unable to cleanup a connection: %s", e) + LOG.warning, "Unable to cleanup a connection: %s", e + ) def _create_connection(self): """Returns a connection instance. @@ -163,9 +168,12 @@ class ConnectionPool(queue.Queue): conn = self.get(timeout=self._connection_get_timeout) except queue.Empty: raise exception.QueueEmpty( - _('Unable to get a connection from pool id %(id)s after ' - '%(seconds)s seconds.') % - {'id': id(self), 'seconds': self._connection_get_timeout}) + _( + 'Unable to get a connection from pool id %(id)s after ' + '%(seconds)s seconds.' + ) + % {'id': id(self), 'seconds': self._connection_get_timeout} + ) self._trace_logger('Acquired connection %s', id(conn)) try: yield conn @@ -218,10 +226,12 @@ class ConnectionPool(queue.Queue): pass def _put(self, conn): - self.queue.append(_PoolItem( - ttl=time.time() + self._unused_timeout, - connection=conn, - )) + self.queue.append( + _PoolItem( + ttl=time.time() + self._unused_timeout, + connection=conn, + ) + ) self._acquired -= 1 @@ -234,10 +244,12 @@ class MemcacheClientPool(ConnectionPool): self._arguments = { 'dead_retry': arguments.get('dead_retry', 5 * 60), 'socket_timeout': arguments.get('socket_timeout', 3.0), - 'server_max_value_length': - arguments.get('server_max_value_length'), + 'server_max_value_length': arguments.get( + 'server_max_value_length' + ), 'flush_on_reconnect': arguments.get( - 'pool_flush_on_reconnect', False), + 'pool_flush_on_reconnect', False + ), } # NOTE(morganfainberg): The host objects expect an int for the # deaduntil value. Initialize this at 0 for each host with 0 indicating @@ -284,7 +296,9 @@ class MemcacheClientPool(ConnectionPool): self._hosts_deaduntil[i] = host.deaduntil self._debug_logger( 'Marked host %s dead until %s', - self.urls[i], host.deaduntil) + self.urls[i], + host.deaduntil, + ) else: self._hosts_deaduntil[i] = 0 finally: diff --git a/oslo_cache/_opts.py b/oslo_cache/_opts.py index 3f5ed565..01bd3689 100644 --- a/oslo_cache/_opts.py +++ b/oslo_cache/_opts.py @@ -19,253 +19,385 @@ _DEFAULT_BACKEND = 'dogpile.cache.null' FILE_OPTIONS = { 'cache': [ - cfg.StrOpt('config_prefix', default='cache.oslo', - help='Prefix for building the configuration dictionary ' - 'for the cache region. This should not need to be ' - 'changed unless there is another dogpile.cache ' - 'region with the same configuration name.'), - cfg.IntOpt('expiration_time', default=600, - min=1, - help='Default TTL, in seconds, for any cached item in ' - 'the dogpile.cache region. This applies to any ' - 'cached method that doesn\'t have an explicit ' - 'cache expiration time defined for it.'), - cfg.IntOpt('backend_expiration_time', - min=1, - help='Expiration time in cache backend to purge ' - 'expired records automatically. This should be ' - 'greater than expiration_time and all cache_time ' - 'options'), + cfg.StrOpt( + 'config_prefix', + default='cache.oslo', + help=( + 'Prefix for building the configuration dictionary ' + 'for the cache region. This should not need to be ' + 'changed unless there is another dogpile.cache ' + 'region with the same configuration name.' + ), + ), + cfg.IntOpt( + 'expiration_time', + default=600, + min=1, + help=( + 'Default TTL, in seconds, for any cached item in ' + 'the dogpile.cache region. This applies to any ' + 'cached method that doesn\'t have an explicit ' + 'cache expiration time defined for it.' + ), + ), + cfg.IntOpt( + 'backend_expiration_time', + min=1, + help=( + 'Expiration time in cache backend to purge ' + 'expired records automatically. This should be ' + 'greater than expiration_time and all cache_time ' + 'options' + ), + ), # NOTE(morganfainberg): It is recommended that either Redis or # Memcached are used as the dogpile backend for real workloads. To # prevent issues with the memory cache ending up in "production" # unintentionally, we register a no-op as the default caching backend. - cfg.StrOpt('backend', default=_DEFAULT_BACKEND, - choices=['oslo_cache.memcache_pool', - 'oslo_cache.dict', - 'oslo_cache.etcd3gw', - 'dogpile.cache.pymemcache', - 'dogpile.cache.memcached', - 'dogpile.cache.pylibmc', - 'dogpile.cache.bmemcached', - 'dogpile.cache.dbm', - 'dogpile.cache.redis', - 'dogpile.cache.redis_sentinel', - 'dogpile.cache.memory', - 'dogpile.cache.memory_pickle', - 'dogpile.cache.null'], - help='Cache backend module. For eventlet-based or ' - 'environments with hundreds of threaded servers, Memcache ' - 'with pooling (oslo_cache.memcache_pool) is recommended. ' - 'For environments with less than 100 threaded servers, ' - 'Memcached (dogpile.cache.memcached) or Redis ' - '(dogpile.cache.redis) is recommended. Test environments ' - 'with a single instance of the server can use the ' - 'dogpile.cache.memory backend.'), - cfg.MultiStrOpt('backend_argument', default=[], secret=True, - help='Arguments supplied to the backend module. ' - 'Specify this option once per argument to be ' - 'passed to the dogpile.cache backend. Example ' - 'format: ":".'), - cfg.ListOpt('proxies', default=[], - help='Proxy classes to import that will affect the way ' - 'the dogpile.cache backend functions. See the ' - 'dogpile.cache documentation on ' - 'changing-backend-behavior.'), - cfg.BoolOpt('enabled', default=False, - help='Global toggle for caching.'), - cfg.BoolOpt('debug_cache_backend', default=False, - help='Extra debugging from the cache backend (cache ' - 'keys, get/set/delete/etc calls). This is only ' - 'really useful if you need to see the specific ' - 'cache-backend get/set/delete calls with the ' - 'keys/values. Typically this should be left set ' - 'to false.'), - cfg.ListOpt('memcache_servers', default=['localhost:11211'], - help='Memcache servers in the format of "host:port". ' - 'This is used by backends dependent on Memcached.' - 'If ``dogpile.cache.memcached`` or ' - '``oslo_cache.memcache_pool`` is used and a given ' - 'host refer to an IPv6 or a given domain refer to ' - 'IPv6 then you should prefix the given address with ' - 'the address family (``inet6``) ' - '(e.g ``inet6:[::1]:11211``, ' - '``inet6:[fd12:3456:789a:1::1]:11211``, ' - '``inet6:[controller-0.internalapi]:11211``). ' - 'If the address family is not given then these ' - 'backends will use the default ``inet`` address ' - 'family which corresponds to IPv4'), - cfg.IntOpt('memcache_dead_retry', - default=5 * 60, - help='Number of seconds memcached server is considered dead' - ' before it is tried again. (dogpile.cache.memcache and' - ' oslo_cache.memcache_pool backends only).'), - cfg.FloatOpt('memcache_socket_timeout', - default=1.0, - help='Timeout in seconds for every call to a server.' - ' (dogpile.cache.memcache and oslo_cache.memcache_pool' - ' backends only).'), - cfg.IntOpt('memcache_pool_maxsize', - default=10, - help='Max total number of open connections to every' - ' memcached server. (oslo_cache.memcache_pool backend' - ' only).'), - cfg.IntOpt('memcache_pool_unused_timeout', - default=60, - help='Number of seconds a connection to memcached is held' - ' unused in the pool before it is closed.' - ' (oslo_cache.memcache_pool backend only).'), - cfg.IntOpt('memcache_pool_connection_get_timeout', - default=10, - help='Number of seconds that an operation will wait to get ' - 'a memcache client connection.'), - cfg.BoolOpt('memcache_pool_flush_on_reconnect', - default=False, - help='Global toggle if memcache will be flushed' - ' on reconnect.' - ' (oslo_cache.memcache_pool backend only).'), - cfg.BoolOpt('memcache_sasl_enabled', - default=False, - help='Enable the SASL(Simple Authentication and Security' - 'Layer) if the SASL_enable is true, else disable.'), - cfg.StrOpt('memcache_username', - help='the user name for the memcached which SASL enabled'), - cfg.StrOpt('memcache_password', - secret=True, - help='the password for the memcached which SASL enabled'), - cfg.StrOpt('redis_server', - default='localhost:6379', - help='Redis server in the format of "host:port"'), - cfg.IntOpt('redis_db', - default=0, - min=0, - help='Database id in Redis server'), - cfg.StrOpt('redis_username', - help='the user name for redis'), - cfg.StrOpt('redis_password', - secret=True, - help='the password for redis'), - cfg.ListOpt('redis_sentinels', - default=['localhost:26379'], - help='Redis sentinel servers in the format of ' - '"host:port"'), - cfg.FloatOpt('redis_socket_timeout', - default=1.0, - help='Timeout in seconds for every call to a server.' - ' (dogpile.cache.redis and dogpile.cache.redis_sentinel ' - 'backends only).'), - cfg.StrOpt('redis_sentinel_service_name', - default='mymaster', - help='Service name of the redis sentinel cluster.'), - cfg.BoolOpt('tls_enabled', - default=False, - help='Global toggle for TLS usage when communicating with' - ' the caching servers. Currently supported by ' - '``dogpile.cache.bmemcache``, ' - '``dogpile.cache.pymemcache``, ' - '``oslo_cache.memcache_pool``, ' - '``dogpile.cache.redis`` and ' - '``dogpile.cache.redis_sentinel``.'), - cfg.StrOpt('tls_cafile', - default=None, - help='Path to a file of concatenated CA certificates in PEM' - ' format necessary to establish the caching servers\'' - ' authenticity. If tls_enabled is False, this option is' - ' ignored.'), - cfg.StrOpt('tls_certfile', - default=None, - help='Path to a single file in PEM format containing the' - ' client\'s certificate as well as any number of CA' - ' certificates needed to establish the certificate\'s' - ' authenticity. This file is only required when client side' - ' authentication is necessary. If tls_enabled is False,' - ' this option is ignored.'), - cfg.StrOpt('tls_keyfile', - default=None, - help='Path to a single file containing the client\'s' - ' private key in. Otherwise the private key will be taken' - ' from the file specified in tls_certfile. If tls_enabled' - ' is False, this option is ignored.'), - cfg.StrOpt('tls_allowed_ciphers', - default=None, - help='Set the available ciphers for sockets created with' - ' the TLS context. It should be a string in the OpenSSL' - ' cipher list format. If not specified, all OpenSSL enabled' - ' ciphers will be available. Currently supported by ' - '``dogpile.cache.bmemcache``, ' - '``dogpile.cache.pymemcache`` and ' - '``oslo_cache.memcache_pool``.'), + cfg.StrOpt( + 'backend', + default=_DEFAULT_BACKEND, + choices=[ + 'oslo_cache.memcache_pool', + 'oslo_cache.dict', + 'oslo_cache.etcd3gw', + 'dogpile.cache.pymemcache', + 'dogpile.cache.memcached', + 'dogpile.cache.pylibmc', + 'dogpile.cache.bmemcached', + 'dogpile.cache.dbm', + 'dogpile.cache.redis', + 'dogpile.cache.redis_sentinel', + 'dogpile.cache.memory', + 'dogpile.cache.memory_pickle', + 'dogpile.cache.null', + ], + help=( + 'Cache backend module. For eventlet-based or ' + 'environments with hundreds of threaded servers, Memcache ' + 'with pooling (oslo_cache.memcache_pool) is recommended. ' + 'For environments with less than 100 threaded servers, ' + 'Memcached (dogpile.cache.memcached) or Redis ' + '(dogpile.cache.redis) is recommended. Test environments ' + 'with a single instance of the server can use the ' + 'dogpile.cache.memory backend.' + ), + ), + cfg.MultiStrOpt( + 'backend_argument', + default=[], + secret=True, + help=( + 'Arguments supplied to the backend module. ' + 'Specify this option once per argument to be ' + 'passed to the dogpile.cache backend. Example ' + 'format: ":".' + ), + ), + cfg.ListOpt( + 'proxies', + default=[], + help=( + 'Proxy classes to import that will affect the way ' + 'the dogpile.cache backend functions. See the ' + 'dogpile.cache documentation on ' + 'changing-backend-behavior.' + ), + ), + cfg.BoolOpt( + 'enabled', default=False, help='Global toggle for caching.' + ), + cfg.BoolOpt( + 'debug_cache_backend', + default=False, + help=( + 'Extra debugging from the cache backend (cache ' + 'keys, get/set/delete/etc calls). This is only ' + 'really useful if you need to see the specific ' + 'cache-backend get/set/delete calls with the ' + 'keys/values. Typically this should be left set ' + 'to false.' + ), + ), + cfg.ListOpt( + 'memcache_servers', + default=['localhost:11211'], + help=( + 'Memcache servers in the format of "host:port". ' + 'This is used by backends dependent on Memcached.' + 'If ``dogpile.cache.memcached`` or ' + '``oslo_cache.memcache_pool`` is used and a given ' + 'host refer to an IPv6 or a given domain refer to ' + 'IPv6 then you should prefix the given address with ' + 'the address family (``inet6``) ' + '(e.g ``inet6:[::1]:11211``, ' + '``inet6:[fd12:3456:789a:1::1]:11211``, ' + '``inet6:[controller-0.internalapi]:11211``). ' + 'If the address family is not given then these ' + 'backends will use the default ``inet`` address ' + 'family which corresponds to IPv4' + ), + ), + cfg.IntOpt( + 'memcache_dead_retry', + default=5 * 60, + help=( + 'Number of seconds memcached server is considered dead ' + 'before it is tried again. (dogpile.cache.memcache and ' + 'oslo_cache.memcache_pool backends only).' + ), + ), + cfg.FloatOpt( + 'memcache_socket_timeout', + default=1.0, + help=( + 'Timeout in seconds for every call to a server. ' + '(dogpile.cache.memcache and oslo_cache.memcache_pool ' + 'backends only).' + ), + ), + cfg.IntOpt( + 'memcache_pool_maxsize', + default=10, + help=( + 'Max total number of open connections to every ' + 'memcached server. (oslo_cache.memcache_pool backend ' + 'only).' + ), + ), + cfg.IntOpt( + 'memcache_pool_unused_timeout', + default=60, + help=( + 'Number of seconds a connection to memcached is held ' + 'unused in the pool before it is closed. ' + '(oslo_cache.memcache_pool backend only).' + ), + ), + cfg.IntOpt( + 'memcache_pool_connection_get_timeout', + default=10, + help=( + 'Number of seconds that an operation will wait to get ' + 'a memcache client connection.' + ), + ), + cfg.BoolOpt( + 'memcache_pool_flush_on_reconnect', + default=False, + help=( + 'Global toggle if memcache will be flushed ' + 'on reconnect. ' + '(oslo_cache.memcache_pool backend only).' + ), + ), + cfg.BoolOpt( + 'memcache_sasl_enabled', + default=False, + help=( + 'Enable the SASL(Simple Authentication and Security' + 'Layer) if the SASL_enable is true, else disable.' + ), + ), + cfg.StrOpt( + 'memcache_username', + help='the user name for the memcached which SASL enabled', + ), + cfg.StrOpt( + 'memcache_password', + secret=True, + help='the password for the memcached which SASL enabled', + ), + cfg.StrOpt( + 'redis_server', + default='localhost:6379', + help='Redis server in the format of "host:port"', + ), + cfg.IntOpt( + 'redis_db', default=0, min=0, help='Database id in Redis server' + ), + cfg.StrOpt('redis_username', help='the user name for redis'), + cfg.StrOpt( + 'redis_password', secret=True, help='the password for redis' + ), + cfg.ListOpt( + 'redis_sentinels', + default=['localhost:26379'], + help='Redis sentinel servers in the format of "host:port"', + ), + cfg.FloatOpt( + 'redis_socket_timeout', + default=1.0, + help=( + 'Timeout in seconds for every call to a server. ' + '(dogpile.cache.redis and dogpile.cache.redis_sentinel ' + 'backends only).' + ), + ), + cfg.StrOpt( + 'redis_sentinel_service_name', + default='mymaster', + help='Service name of the redis sentinel cluster.', + ), + cfg.BoolOpt( + 'tls_enabled', + default=False, + help=( + 'Global toggle for TLS usage when communicating with' + ' the caching servers. Currently supported by ' + '``dogpile.cache.bmemcache``, ' + '``dogpile.cache.pymemcache``, ' + '``oslo_cache.memcache_pool``, ' + '``dogpile.cache.redis`` and ' + '``dogpile.cache.redis_sentinel``.' + ), + ), + cfg.StrOpt( + 'tls_cafile', + default=None, + help=( + 'Path to a file of concatenated CA certificates in PEM ' + 'format necessary to establish the caching servers\' ' + 'authenticity. If tls_enabled is False, this option is ' + 'ignored.' + ), + ), + cfg.StrOpt( + 'tls_certfile', + default=None, + help=( + 'Path to a single file in PEM format containing the ' + 'client\'s certificate as well as any number of CA ' + 'certificates needed to establish the certificate\'s ' + 'authenticity. This file is only required when client side ' + 'authentication is necessary. If tls_enabled is False, ' + 'this option is ignored.' + ), + ), + cfg.StrOpt( + 'tls_keyfile', + default=None, + help=( + 'Path to a single file containing the client\'s ' + 'private key in. Otherwise the private key will be taken ' + 'from the file specified in tls_certfile. If tls_enabled ' + 'is False, this option is ignored.' + ), + ), + cfg.StrOpt( + 'tls_allowed_ciphers', + default=None, + help=( + 'Set the available ciphers for sockets created with ' + 'the TLS context. It should be a string in the OpenSSL ' + 'cipher list format. If not specified, all OpenSSL enabled ' + 'ciphers will be available. Currently supported by ' + '``dogpile.cache.bmemcache``, ' + '``dogpile.cache.pymemcache`` and ' + '``oslo_cache.memcache_pool``.' + ), + ), cfg.BoolOpt( 'enable_socket_keepalive', default=False, - help="Global toggle for the socket keepalive of " - "dogpile's pymemcache backend"), + help=( + "Global toggle for the socket keepalive of " + "dogpile's pymemcache backend" + ), + ), cfg.IntOpt( 'socket_keepalive_idle', default=1, min=0, - help='The time (in seconds) the connection needs to ' - 'remain idle before TCP starts sending keepalive probes. ' - 'Should be a positive integer most greater than zero.'), + help=( + 'The time (in seconds) the connection needs to ' + 'remain idle before TCP starts sending keepalive probes. ' + 'Should be a positive integer most greater than zero.' + ), + ), cfg.IntOpt( 'socket_keepalive_interval', default=1, min=0, - help='The time (in seconds) between individual keepalive ' - 'probes. Should be a positive integer greater ' - 'than zero.'), + help=( + 'The time (in seconds) between individual keepalive ' + 'probes. Should be a positive integer greater ' + 'than zero.' + ), + ), cfg.IntOpt( 'socket_keepalive_count', default=1, min=0, - help='The maximum number of keepalive probes TCP should ' - 'send before dropping the connection. Should be a ' - 'positive integer greater than zero.'), + help=( + 'The maximum number of keepalive probes TCP should ' + 'send before dropping the connection. Should be a ' + 'positive integer greater than zero.' + ), + ), cfg.BoolOpt( 'enable_retry_client', default=False, - help='Enable retry client mechanisms to handle failure. ' - 'Those mechanisms can be used to wrap all kind of pymemcache ' - 'clients. The wrapper allows you to define how many attempts ' - 'to make and how long to wait between attempts.'), + help=( + 'Enable retry client mechanisms to handle failure. ' + 'Those mechanisms can be used to wrap all kind of pymemcache ' + 'clients. The wrapper allows you to define how many attempts ' + 'to make and how long to wait between attempts.' + ), + ), cfg.IntOpt( 'retry_attempts', min=1, default=2, - help='Number of times to attempt an action before failing.'), + help='Number of times to attempt an action before failing.', + ), cfg.FloatOpt( 'retry_delay', default=0, - help='Number of seconds to sleep between each attempt.'), + help='Number of seconds to sleep between each attempt.', + ), cfg.IntOpt( 'hashclient_retry_attempts', min=1, default=2, - help='Amount of times a client should be tried ' - 'before it is marked dead and removed from the pool in ' - 'the HashClient\'s internal mechanisms.'), + help=( + 'Amount of times a client should be tried ' + 'before it is marked dead and removed from the pool in ' + 'the HashClient\'s internal mechanisms.' + ), + ), cfg.FloatOpt( 'hashclient_retry_delay', default=1, - help='Time in seconds that should pass between ' - 'retry attempts in the HashClient\'s internal mechanisms.'), + help=( + 'Time in seconds that should pass between ' + 'retry attempts in the HashClient\'s internal mechanisms.' + ), + ), cfg.FloatOpt( 'dead_timeout', default=60, - help='Time in seconds before attempting to add a node ' - 'back in the pool in the HashClient\'s internal mechanisms.'), - cfg.BoolOpt('enforce_fips_mode', - default=False, - help='Global toggle for enforcing the OpenSSL FIPS mode. ' - 'This feature requires Python support. ' - 'This is available in Python 3.9 in all ' - 'environments and may have been backported to older ' - 'Python versions on select environments. If the Python ' - 'executable used does not support OpenSSL FIPS mode, ' - 'an exception will be raised. Currently supported by ' - '``dogpile.cache.bmemcache``, ' - '``dogpile.cache.pymemcache`` and ' - '``oslo_cache.memcache_pool``.'), + help=( + 'Time in seconds before attempting to add a node ' + 'back in the pool in the HashClient\'s internal mechanisms.' + ), + ), + cfg.BoolOpt( + 'enforce_fips_mode', + default=False, + help=( + 'Global toggle for enforcing the OpenSSL FIPS mode. ' + 'This feature requires Python support. ' + 'This is available in Python 3.9 in all ' + 'environments and may have been backported to older ' + 'Python versions on select environments. If the Python ' + 'executable used does not support OpenSSL FIPS mode, ' + 'an exception will be raised. Currently supported by ' + '``dogpile.cache.bmemcache``, ' + '``dogpile.cache.pymemcache`` and ' + '``oslo_cache.memcache_pool``.' + ), + ), ], } @@ -292,7 +424,8 @@ def set_defaults(conf, memcache_pool_flush_on_reconnect=False): cfg.set_defaults( FILE_OPTIONS, - memcache_pool_flush_on_reconnect=memcache_pool_flush_on_reconnect) + memcache_pool_flush_on_reconnect=memcache_pool_flush_on_reconnect, + ) def list_opts(): diff --git a/oslo_cache/backends/dictionary.py b/oslo_cache/backends/dictionary.py index 352b63d3..3626b489 100644 --- a/oslo_cache/backends/dictionary.py +++ b/oslo_cache/backends/dictionary.py @@ -18,9 +18,7 @@ from dogpile.cache import api from oslo_cache import core from oslo_utils import timeutils -__all__ = [ - 'DictCacheBackend' -] +__all__ = ['DictCacheBackend'] _NO_VALUE = core.NO_VALUE diff --git a/oslo_cache/backends/etcd3gw.py b/oslo_cache/backends/etcd3gw.py index 05b1f8e9..3c70fcb3 100644 --- a/oslo_cache/backends/etcd3gw.py +++ b/oslo_cache/backends/etcd3gw.py @@ -13,14 +13,13 @@ # under the License. """dogpile.cache backend that uses etcd 3.x for storage""" + from dogpile.cache import api from oslo_cache import core from oslo_serialization import jsonutils -__all__ = [ - 'Etcd3gwCacheBackend' -] +__all__ = ['Etcd3gwCacheBackend'] _NO_VALUE = core.NO_VALUE @@ -41,9 +40,10 @@ class Etcd3gwCacheBackend(api.CacheBackend): self.timeout = int(arguments.get('timeout', self.DEFAULT_TIMEOUT)) # module etcd3gw is only required when etcd3gw backend is used import etcd3gw - self._client = etcd3gw.client(host=self.host, - port=self.port, - timeout=self.timeout) + + self._client = etcd3gw.client( + host=self.host, port=self.port, timeout=self.timeout + ) def get(self, key): values = self._client.get(key, False) diff --git a/oslo_cache/backends/memcache_pool.py b/oslo_cache/backends/memcache_pool.py index b161e0b5..acdb65c9 100644 --- a/oslo_cache/backends/memcache_pool.py +++ b/oslo_cache/backends/memcache_pool.py @@ -63,14 +63,17 @@ class PooledMemcachedBackend(memcached_backend.MemcachedBackend): # Composed from GenericMemcachedBackend's and MemcacheArgs's __init__ def __init__(self, arguments): super().__init__(arguments) - if (arguments.get('tls_enabled', False) or - arguments.get('sasl_enabled', False)): - if (arguments.get('sasl_enabled', False) and - (arguments.get('username') is None or - arguments.get('password') is None)): + if arguments.get('tls_enabled', False) or arguments.get( + 'sasl_enabled', False + ): + if arguments.get('sasl_enabled', False) and ( + arguments.get('username') is None + or arguments.get('password') is None + ): raise exception.ConfigurationError( 'username and password should be configured to use SASL ' - 'authentication.') + 'authentication.' + ) if not _bmemcache_pool: raise ImportError("python-binary-memcached package is missing") self.client_pool = _bmemcache_pool.BMemcacheClientPool( @@ -78,8 +81,9 @@ class PooledMemcachedBackend(memcached_backend.MemcachedBackend): arguments, maxsize=arguments.get('pool_maxsize', 10), unused_timeout=arguments.get('pool_unused_timeout', 60), - conn_get_timeout=arguments.get('pool_connection_get_timeout', - 10), + conn_get_timeout=arguments.get( + 'pool_connection_get_timeout', 10 + ), ) else: self.client_pool = _memcache_pool.MemcacheClientPool( @@ -87,8 +91,9 @@ class PooledMemcachedBackend(memcached_backend.MemcachedBackend): arguments, maxsize=arguments.get('pool_maxsize', 10), unused_timeout=arguments.get('pool_unused_timeout', 60), - conn_get_timeout=arguments.get('pool_connection_get_timeout', - 10), + conn_get_timeout=arguments.get( + 'pool_connection_get_timeout', 10 + ), ) # Since all methods in backend just call one of methods of client, this diff --git a/oslo_cache/core.py b/oslo_cache/core.py index 8cb9035c..84ad5f60 100644 --- a/oslo_cache/core.py +++ b/oslo_cache/core.py @@ -32,8 +32,10 @@ The library has special public value for nonexistent or expired keys called :data:`NO_VALUE`. To use this value you should import it from oslo_cache.core:: from oslo_cache import core + NO_VALUE = core.NO_VALUE """ + import socket import ssl import urllib.parse @@ -67,6 +69,7 @@ _LOG = log.getLogger(__name__) class _DebugProxy(proxy.ProxyBackend): """Extra Logging ProxyBackend.""" + # NOTE(morganfainberg): Pass all key/values through repr to ensure we have # a clean description of the information. Without use of repr, it might # be possible to run into encode/decode error(s). For logging/debugging @@ -75,19 +78,25 @@ class _DebugProxy(proxy.ProxyBackend): def get(self, key): value = self.proxied.get(key) - _LOG.debug('CACHE_GET: Key: "%(key)r" Value: "%(value)r"', - {'key': key, 'value': value}) + _LOG.debug( + 'CACHE_GET: Key: "%(key)r" Value: "%(value)r"', + {'key': key, 'value': value}, + ) return value def get_multi(self, keys): values = self.proxied.get_multi(keys) - _LOG.debug('CACHE_GET_MULTI: "%(keys)r" Values: "%(values)r"', - {'keys': keys, 'values': values}) + _LOG.debug( + 'CACHE_GET_MULTI: "%(keys)r" Values: "%(values)r"', + {'keys': keys, 'values': values}, + ) return values def set(self, key, value): - _LOG.debug('CACHE_SET: Key: "%(key)r" Value: "%(value)r"', - {'key': key, 'value': value}) + _LOG.debug( + 'CACHE_SET: Key: "%(key)r" Value: "%(value)r"', + {'key': key, 'value': value}, + ) return self.proxied.set(key, value) def set_multi(self, keys): @@ -125,8 +134,10 @@ def _build_cache_config(conf): try: (argname, argvalue) = argument.split(':', 1) except ValueError: - msg = ('Unable to build cache config-key. Expected format ' - '":". Skipping unknown format: %s') + msg = ( + 'Unable to build cache config-key. Expected format ' + '":". Skipping unknown format: %s' + ) _LOG.error(msg, argument) continue @@ -139,9 +150,11 @@ def _build_cache_config(conf): # take data and do not handle processing/validation as expected # directly makes for odd behaviors when wrapping dogpile.cache in # a library like oslo.cache - if (conf.cache.backend - in ('dogpile.cache.memcached', 'oslo_cache.memcache_pool') and - argname == 'url'): + if ( + conf.cache.backend + in ('dogpile.cache.memcached', 'oslo_cache.memcache_pool') + and argname == 'url' + ): argvalue = argvalue.split(',') conf_dict[arg_key] = argvalue @@ -152,21 +165,23 @@ def _build_cache_config(conf): netloc = conf.cache.redis_server else: if conf.cache.redis_username: - netloc = '{}:{}@{}'.format(conf.cache.redis_username, - conf.cache.redis_password, - conf.cache.redis_server) + netloc = f'{conf.cache.redis_username}:{conf.cache.redis_password}@{conf.cache.redis_server}' else: - netloc = ':{}@{}'.format(conf.cache.redis_password, - conf.cache.redis_server) + netloc = ( + f':{conf.cache.redis_password}@{conf.cache.redis_server}' + ) parts = urllib.parse.ParseResult( scheme=('rediss' if conf.cache.tls_enabled else 'redis'), - netloc=netloc, path=str(conf.cache.redis_db), params='', query='', - fragment='') + netloc=netloc, + path=str(conf.cache.redis_db), + params='', + query='', + fragment='', + ) conf_dict.setdefault( - f'{prefix}.arguments.url', - urllib.parse.urlunparse(parts) + f'{prefix}.arguments.url', urllib.parse.urlunparse(parts) ) for arg in ('socket_timeout',): value = getattr(conf.cache, 'redis_' + arg) @@ -175,11 +190,13 @@ def _build_cache_config(conf): for arg in ('username', 'password', 'socket_timeout', 'db'): value = getattr(conf.cache, 'redis_' + arg) conf_dict[f'{prefix}.arguments.{arg}'] = value - conf_dict[f'{prefix}.arguments.service_name'] = \ + conf_dict[f'{prefix}.arguments.service_name'] = ( conf.cache.redis_sentinel_service_name + ) if conf.cache.redis_sentinels: conf_dict[f'{prefix}.arguments.sentinels'] = [ - _parse_sentinel(s) for s in conf.cache.redis_sentinels] + _parse_sentinel(s) for s in conf.cache.redis_sentinels + ] else: # NOTE(yorik-sar): these arguments will be used for memcache-related # backends. Use setdefault for url to support old-style setting through @@ -204,13 +221,21 @@ def _build_cache_config(conf): # # The normal non-pooled clients connect explicitly on each use and # does not need the explicit flush_on_reconnect - conf_dict.setdefault(f'{prefix}.arguments.url', - conf.cache.memcache_servers) + conf_dict.setdefault( + f'{prefix}.arguments.url', conf.cache.memcache_servers + ) - for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize', - 'pool_unused_timeout', 'pool_connection_get_timeout', - 'pool_flush_on_reconnect', 'sasl_enabled', 'username', - 'password'): + for arg in ( + 'dead_retry', + 'socket_timeout', + 'pool_maxsize', + 'pool_unused_timeout', + 'pool_connection_get_timeout', + 'pool_flush_on_reconnect', + 'sasl_enabled', + 'username', + 'password', + ): value = getattr(conf.cache, 'memcache_' + arg) conf_dict[f'{prefix}.arguments.{arg}'] = value @@ -218,29 +243,40 @@ def _build_cache_config(conf): if conf.cache.expiration_time > conf.cache.backend_expiration_time: raise exception.ConfigurationError( "backend_expiration_time should not be smaller than " - "expiration_time.") - if conf.cache.backend in ('dogpile.cache.pymemcache', - 'dogpile.cache.memcached', - 'dogpile.cache.pylibmc', - 'oslo_cache.memcache_pool'): - conf_dict[f'{prefix}.arguments.memcached_expire_time'] = \ + "expiration_time." + ) + if conf.cache.backend in ( + 'dogpile.cache.pymemcache', + 'dogpile.cache.memcached', + 'dogpile.cache.pylibmc', + 'oslo_cache.memcache_pool', + ): + conf_dict[f'{prefix}.arguments.memcached_expire_time'] = ( conf.cache.backend_expiration_time - elif conf.cache.backend in ('dogpile.cache.redis', - 'dogpile.cache.redis_sentinel'): - conf_dict[f'{prefix}.arguments.redis_expiration_time'] = \ + ) + elif conf.cache.backend in ( + 'dogpile.cache.redis', + 'dogpile.cache.redis_sentinel', + ): + conf_dict[f'{prefix}.arguments.redis_expiration_time'] = ( conf.cache.backend_expiration_time + ) else: raise exception.ConfigurationError( - "Enabling backend expiration is not supported by" - "the %s driver", conf.cache.backend) + "Enabling backend expiration is not supported bythe %s driver", + conf.cache.backend, + ) if conf.cache.tls_enabled: - if conf.cache.backend in ('dogpile.cache.bmemcache', - 'dogpile.cache.pymemcache', - 'oslo_cache.memcache_pool'): + if conf.cache.backend in ( + 'dogpile.cache.bmemcache', + 'dogpile.cache.pymemcache', + 'oslo_cache.memcache_pool', + ): _LOG.debug('Oslo Cache TLS - CA: %s', conf.cache.tls_cafile) tls_context = ssl.create_default_context( - cafile=conf.cache.tls_cafile) + cafile=conf.cache.tls_cafile + ) if conf.cache.enforce_fips_mode: if hasattr(ssl, 'FIPS_mode'): @@ -253,11 +289,13 @@ def _build_cache_config(conf): "executable used to a version with FIPS mode support " "or disable FIPS mode by setting " "the '[cache] enforce_fips_mode' configuration option " - "to 'False'.") + "to 'False'." + ) if conf.cache.tls_certfile is not None: - _LOG.debug('Oslo Cache TLS - cert: %s', - conf.cache.tls_certfile) + _LOG.debug( + 'Oslo Cache TLS - cert: %s', conf.cache.tls_certfile + ) _LOG.debug('Oslo Cache TLS - key: %s', conf.cache.tls_keyfile) tls_context.load_cert_chain( conf.cache.tls_certfile, @@ -274,45 +312,53 @@ def _build_cache_config(conf): conf_dict[f'{prefix}.arguments.tls_context'] = tls_context # pass the value of tls_enabled to the backend - conf_dict[f'{prefix}.arguments.tls_enabled'] = \ + conf_dict[f'{prefix}.arguments.tls_enabled'] = ( conf.cache.tls_enabled - elif conf.cache.backend in ('dogpile.cache.redis', - 'dogpile.cache.redis_sentinel'): + ) + elif conf.cache.backend in ( + 'dogpile.cache.redis', + 'dogpile.cache.redis_sentinel', + ): if conf.cache.tls_allowed_ciphers is not None: raise exception.ConfigurationError( "Limiting allowed ciphers is not supported by " - "the %s backend" % conf.cache.backend) + f"the {conf.cache.backend} backend" + ) if conf.cache.enforce_fips_mode: raise exception.ConfigurationError( - "FIPS mode is not supported by the %s backend" % - conf.cache.backend) + f"FIPS mode is not supported by the {conf.cache.backend} backend" + ) conn_kwargs = {} if conf.cache.tls_cafile is not None: _LOG.debug('Oslo Cache TLS - CA: %s', conf.cache.tls_cafile) conn_kwargs['ssl_ca_certs'] = conf.cache.tls_cafile if conf.cache.tls_certfile is not None: - _LOG.debug('Oslo Cache TLS - cert: %s', - conf.cache.tls_certfile) + _LOG.debug( + 'Oslo Cache TLS - cert: %s', conf.cache.tls_certfile + ) _LOG.debug('Oslo Cache TLS - key: %s', conf.cache.tls_keyfile) - conn_kwargs.update({ - 'ssl_certfile': conf.cache.tls_certfile, - 'ssl_keyfile': conf.cache.tls_keyfile - }) + conn_kwargs.update( + { + 'ssl_certfile': conf.cache.tls_certfile, + 'ssl_keyfile': conf.cache.tls_keyfile, + } + ) if conf.cache.backend == 'dogpile.cache.redis_sentinel': conn_kwargs.update({'ssl': True}) - conf_dict[f'{prefix}.arguments.connection_kwargs'] = \ - conn_kwargs - conf_dict[f'{prefix}.arguments.sentinel_kwargs'] = \ + conf_dict[f'{prefix}.arguments.connection_kwargs'] = ( conn_kwargs + ) + conf_dict[f'{prefix}.arguments.sentinel_kwargs'] = conn_kwargs else: - conf_dict[f'{prefix}.arguments.connection_kwargs'] = \ + conf_dict[f'{prefix}.arguments.connection_kwargs'] = ( conn_kwargs + ) else: raise exception.ConfigurationError( "TLS setting via [cache] tls_enabled is not supported by the " - "%s backend. Set [cache] tls_enabled=False or use a different " - "backend." % conf.cache.backend + f"{conf.cache.backend} backend. Set [cache] tls_enabled=False or use a different " + "backend." ) # NOTE(hberaud): Pymemcache backend and redis backends support socket @@ -325,32 +371,38 @@ def _build_cache_config(conf): if conf.cache.enable_socket_keepalive: if conf.cache.backend == 'dogpile.cache.pymemcache': import pymemcache + socket_keepalive = pymemcache.KeepaliveOpts( idle=conf.cache.socket_keepalive_idle, intvl=conf.cache.socket_keepalive_interval, - cnt=conf.cache.socket_keepalive_count) + cnt=conf.cache.socket_keepalive_count, + ) # As with the TLS context above, the config dict below will be # consumed by dogpile.cache that will be used as a proxy between # oslo.cache and pymemcache. - conf_dict[f'{prefix}.arguments.socket_keepalive'] = \ + conf_dict[f'{prefix}.arguments.socket_keepalive'] = ( socket_keepalive - elif conf.cache.backend in ('dogpile.cache.redis', - 'dogpile.cache.redis_sentinel'): + ) + elif conf.cache.backend in ( + 'dogpile.cache.redis', + 'dogpile.cache.redis_sentinel', + ): socket_keepalive_options = { socket.TCP_KEEPIDLE: conf.cache.socket_keepalive_idle, socket.TCP_KEEPINTVL: conf.cache.socket_keepalive_interval, - socket.TCP_KEEPCNT: conf.cache.socket_keepalive_count + socket.TCP_KEEPCNT: conf.cache.socket_keepalive_count, } conf_dict.setdefault( f'{prefix}.arguments.connection_kwargs', {} - ).update({ - 'socket_keepalive': True, - 'socket_keepalive_options': socket_keepalive_options - }) + ).update( + { + 'socket_keepalive': True, + 'socket_keepalive_options': socket_keepalive_options, + } + ) else: raise exception.ConfigurationError( - "Socket keepalive is not supported by the %s backend" - % conf.cache.backend + f"Socket keepalive is not supported by the {conf.cache.backend} backend" ) # NOTE(hberaud): The pymemcache library comes with retry mechanisms that @@ -366,17 +418,19 @@ def _build_cache_config(conf): ) raise exception.ConfigurationError(msg) import pymemcache + conf_dict[f'{prefix}.arguments.enable_retry_client'] = True - conf_dict[f'{prefix}.arguments.retry_attempts'] = \ + conf_dict[f'{prefix}.arguments.retry_attempts'] = ( conf.cache.retry_attempts - conf_dict[f'{prefix}.arguments.retry_delay'] = \ - conf.cache.retry_delay - conf_dict[f'{prefix}.arguments.hashclient_retry_attempts'] = \ + ) + conf_dict[f'{prefix}.arguments.retry_delay'] = conf.cache.retry_delay + conf_dict[f'{prefix}.arguments.hashclient_retry_attempts'] = ( conf.cache.hashclient_retry_attempts - conf_dict[f'{prefix}.arguments.hashclient_retry_delay'] = \ + ) + conf_dict[f'{prefix}.arguments.hashclient_retry_delay'] = ( conf.cache.hashclient_retry_delay - conf_dict[f'{prefix}.arguments.dead_timeout'] = \ - conf.cache.dead_timeout + ) + conf_dict[f'{prefix}.arguments.dead_timeout'] = conf.cache.dead_timeout return conf_dict @@ -456,7 +510,8 @@ def configure_cache_region(conf, region): """ if not isinstance(region, dogpile.cache.CacheRegion): raise exception.ConfigurationError( - _('region not type dogpile.cache.CacheRegion')) + _('region not type dogpile.cache.CacheRegion') + ) if not region.is_configured: # NOTE(morganfainberg): this is how you tell if a region is configured. @@ -464,8 +519,9 @@ def configure_cache_region(conf, region): # easier / less ugly. config_dict = _build_cache_config(conf) - region.configure_from_config(config_dict, - f'{conf.cache.config_prefix}.') + region.configure_from_config( + config_dict, f'{conf.cache.config_prefix}.' + ) if conf.cache.debug_cache_backend: region.wrap(_DebugProxy) @@ -509,11 +565,13 @@ def _get_should_cache_fn(conf, group): :type group: string :returns: function reference """ + def should_cache(value): if not conf.cache.enabled: return False conf_group = getattr(conf, group) return getattr(conf_group, 'caching', True) + return should_cache @@ -538,9 +596,11 @@ def _get_expiration_time_fn(conf, group): :type group: string :rtype: function reference """ + def get_expiration_time(): conf_group = getattr(conf, group) return getattr(conf_group, 'cache_time', None) + return get_expiration_time @@ -566,19 +626,21 @@ def get_memoization_decorator(conf, region, group, expiration_group=None): import oslo_cache.core MEMOIZE = oslo_cache.core.get_memoization_decorator( - conf, region, group='group1') + conf, region, group='group1' + ) + @MEMOIZE - def function(arg1, arg2): - ... + def function(arg1, arg2): ... ALTERNATE_MEMOIZE = oslo_cache.core.get_memoization_decorator( - conf, region, group='group2', expiration_group='group3') + conf, region, group='group2', expiration_group='group3' + ) + @ALTERNATE_MEMOIZE - def function2(arg1, arg2): - ... + def function2(arg1, arg2): ... :param conf: config object, must have had :func:`configure` called on it. :type conf: oslo_config.cfg.ConfigOpts @@ -598,8 +660,9 @@ def get_memoization_decorator(conf, region, group, expiration_group=None): should_cache = _get_should_cache_fn(conf, group) expiration_time = _get_expiration_time_fn(conf, expiration_group) - memoize = region.cache_on_arguments(should_cache_fn=should_cache, - expiration_time=expiration_time) + memoize = region.cache_on_arguments( + should_cache_fn=should_cache, expiration_time=expiration_time + ) # Make sure the actual "should_cache" and "expiration_time" methods are # available. This is potentially interesting/useful to pre-seed cache diff --git a/oslo_cache/testing.py b/oslo_cache/testing.py index 8abfcff6..2c1d6fc9 100644 --- a/oslo_cache/testing.py +++ b/oslo_cache/testing.py @@ -14,7 +14,6 @@ """Items useful for external testing.""" - import copy from dogpile.cache import proxy @@ -60,9 +59,11 @@ class CacheIsolatingProxy(proxy.ProxyBackend): group='cache', backend='dogpile.cache.memory', enabled=True, - proxies=['oslo_cache.testing.CacheIsolatingProxy']) + proxies=['oslo_cache.testing.CacheIsolatingProxy'], + ) """ + def get(self, key): return _copy_value(self.proxied.get(key)) diff --git a/oslo_cache/tests/functional/dogpile_cache_pymemcache/test_cache_backend.py b/oslo_cache/tests/functional/dogpile_cache_pymemcache/test_cache_backend.py index 0fbb4473..b8e0ef0e 100644 --- a/oslo_cache/tests/functional/dogpile_cache_pymemcache/test_cache_backend.py +++ b/oslo_cache/tests/functional/dogpile_cache_pymemcache/test_cache_backend.py @@ -25,7 +25,7 @@ class TestDogpileCachePyMemcacheBackend(test_base.BaseTestCaseCacheBackend): self.config_fixture.config( group="cache", backend="dogpile.cache.pymemcache", - memcache_servers=[f'localhost:{MEMCACHED_PORT}'] + memcache_servers=[f'localhost:{MEMCACHED_PORT}'], ) # NOTE(hberaud): super must be called after all to ensure that diff --git a/oslo_cache/tests/functional/dogpile_cache_redis_sentinel/test_cache_backend.py b/oslo_cache/tests/functional/dogpile_cache_redis_sentinel/test_cache_backend.py index f46a2d3a..21063142 100644 --- a/oslo_cache/tests/functional/dogpile_cache_redis_sentinel/test_cache_backend.py +++ b/oslo_cache/tests/functional/dogpile_cache_redis_sentinel/test_cache_backend.py @@ -26,7 +26,7 @@ class TestRedisSentinelCacheBackend(test_base.BaseTestCaseCacheBackend): group='cache', backend='dogpile.cache.redis_sentinel', redis_sentinels=[f"127.0.0.1:{REDIS_SENTINEL_PORT}"], - redis_sentinel_service_name='pifpaf' + redis_sentinel_service_name='pifpaf', ) # NOTE(hberaud): super must be called after all to ensure that diff --git a/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py b/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py index 10c510b6..a5a2eaec 100644 --- a/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py +++ b/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py @@ -25,7 +25,7 @@ class TestEtcdCacheBackend(test_base.BaseTestCaseCacheBackend): self.config_fixture.config( group='cache', backend='oslo_cache.etcd3gw', - backend_argument=['host:127.0.0.1', f'port:{ETCD_PORT}'] + backend_argument=['host:127.0.0.1', f'port:{ETCD_PORT}'], ) # NOTE(hberaud): super must be called after all to ensure that diff --git a/oslo_cache/tests/functional/memcache_pool/test_cache_backend.py b/oslo_cache/tests/functional/memcache_pool/test_cache_backend.py index b8661842..553bc83a 100644 --- a/oslo_cache/tests/functional/memcache_pool/test_cache_backend.py +++ b/oslo_cache/tests/functional/memcache_pool/test_cache_backend.py @@ -26,7 +26,7 @@ class TestMemcachePoolCacheBackend(test_base.BaseTestCaseCacheBackend): group='cache', backend='oslo_cache.memcache_pool', enabled=True, - memcache_servers=[f'localhost:{MEMCACHED_PORT}'] + memcache_servers=[f'localhost:{MEMCACHED_PORT}'], ) # NOTE(hberaud): super must be called after all to ensure that # config fixture is properly initialized with value related to @@ -45,6 +45,6 @@ class TestBMemcachePoolCacheBackend(test_base.BaseTestCaseCacheBackend): memcache_servers=[f'localhost:{MEMCACHED_PORT}'], memcache_sasl_enabled=False, memcache_username='sasl_name', - memcache_password='sasl_pswd' + memcache_password='sasl_pswd', ) super().setUp() diff --git a/oslo_cache/tests/functional/test_base.py b/oslo_cache/tests/functional/test_base.py index 0a1ffc99..ac755ae5 100644 --- a/oslo_cache/tests/functional/test_base.py +++ b/oslo_cache/tests/functional/test_base.py @@ -86,9 +86,11 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): random_key1 = uuidutils.generate_uuid(dashed=False) random_key2 = uuidutils.generate_uuid(dashed=False) random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} + mapping = { + random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3', + } self.region.set_multi(mapping) # should return NO_VALUE as key does not exist in cache self.assertEqual(NO_VALUE, self.region.get(random_key)) @@ -102,9 +104,11 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): random_key1 = uuidutils.generate_uuid(dashed=False) random_key2 = uuidutils.generate_uuid(dashed=False) random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: '', - random_key3: 'dummyValue3'} + mapping = { + random_key1: 'dummyValue1', + random_key2: '', + random_key3: 'dummyValue3', + } self.region.set_multi(mapping) keys = [random_key, random_key1, random_key2, random_key3] @@ -120,9 +124,11 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): random_key1 = uuidutils.generate_uuid(dashed=False) random_key2 = uuidutils.generate_uuid(dashed=False) random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} + mapping = { + random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3', + } self.region.set_multi(mapping) # should return NO_VALUE as key does not exist in cache self.assertEqual(NO_VALUE, self.region.get(random_key)) @@ -130,8 +136,7 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): self.assertEqual("dummyValue2", self.region.get(random_key2)) self.assertEqual("dummyValue3", self.region.get(random_key3)) - mapping = {random_key1: 'dummyValue4', - random_key2: 'dummyValue5'} + mapping = {random_key1: 'dummyValue4', random_key2: 'dummyValue5'} self.region.set_multi(mapping) self.assertEqual(NO_VALUE, self.region.get(random_key)) self.assertEqual("dummyValue4", self.region.get(random_key1)) @@ -144,10 +149,12 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): random_key2 = uuidutils.generate_uuid(dashed=False) random_key3 = uuidutils.generate_uuid(dashed=False) random_key4 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: None, - random_key3: '', - random_key4: 'dummyValue4'} + mapping = { + random_key1: 'dummyValue1', + random_key2: None, + random_key3: '', + random_key4: 'dummyValue4', + } self.region.set_multi(mapping) # should return NO_VALUE as key does not exist in cache self.assertEqual(NO_VALUE, self.region.get(random_key)) @@ -166,8 +173,7 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): self.assertEqual("", results[3]) self.assertEqual("dummyValue4", results[4]) - mapping = {random_key1: 'dummyValue5', - random_key2: 'dummyValue6'} + mapping = {random_key1: 'dummyValue5', random_key2: 'dummyValue6'} self.region.set_multi(mapping) self.assertEqual(NO_VALUE, self.region.get(random_key)) self.assertEqual("dummyValue5", self.region.get(random_key1)) @@ -188,9 +194,11 @@ class BaseTestCaseCacheBackend(base.BaseTestCase): random_key1 = uuidutils.generate_uuid(dashed=False) random_key2 = uuidutils.generate_uuid(dashed=False) random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} + mapping = { + random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3', + } self.region.set_multi(mapping) # should return NO_VALUE as key does not exist in cache self.assertEqual(NO_VALUE, self.region.get(random_key)) diff --git a/oslo_cache/tests/test_cache.py b/oslo_cache/tests/test_cache.py index 9cb0f532..b42b87d9 100644 --- a/oslo_cache/tests/test_cache.py +++ b/oslo_cache/tests/test_cache.py @@ -26,4 +26,5 @@ class BaseTestCase(base.BaseTestCase): group='cache', backend='dogpile.cache.memory', enabled=True, - proxies=['oslo_cache.testing.CacheIsolatingProxy']) + proxies=['oslo_cache.testing.CacheIsolatingProxy'], + ) diff --git a/oslo_cache/tests/unit/test_cache_basics.py b/oslo_cache/tests/unit/test_cache_basics.py index dfd612c8..7efd09f1 100644 --- a/oslo_cache/tests/unit/test_cache_basics.py +++ b/oslo_cache/tests/unit/test_cache_basics.py @@ -56,33 +56,38 @@ class TestProxyValue: class CacheRegionTest(test_cache.BaseTestCase): - def setUp(self): super().setUp() self.region = cache.create_region() cache.configure_cache_region(self.config_fixture.conf, self.region) self.region.wrap(TestProxy) self.region_kwargs = cache.create_region( - function=cache.kwarg_function_key_generator) - cache.configure_cache_region(self.config_fixture.conf, - self.region_kwargs) + function=cache.kwarg_function_key_generator + ) + cache.configure_cache_region( + self.config_fixture.conf, self.region_kwargs + ) self.region_kwargs.wrap(TestProxy) self.test_value = TestProxyValue('Decorator Test') def _add_test_caching_option(self): self.config_fixture.register_opt( - cfg.BoolOpt('caching', default=True), group='cache') + cfg.BoolOpt('caching', default=True), group='cache' + ) def _add_dummy_config_group(self): self.config_fixture.register_opt( - cfg.IntOpt('cache_time'), group=TEST_GROUP) + cfg.IntOpt('cache_time'), group=TEST_GROUP + ) self.config_fixture.register_opt( - cfg.IntOpt('cache_time'), group=TEST_GROUP2) + cfg.IntOpt('cache_time'), group=TEST_GROUP2 + ) def _get_cacheable_function(self, region=None): region = region if region else self.region memoize = cache.get_memoization_decorator( - self.config_fixture.conf, region, group='cache') + self.config_fixture.conf, region, group='cache' + ) @memoize def cacheable_function(value=0, **kw): @@ -107,7 +112,8 @@ class CacheRegionTest(test_cache.BaseTestCase): self.config_fixture.conf, self.region, group='cache', - expiration_group=TEST_GROUP2) + expiration_group=TEST_GROUP2, + ) class _test_obj: def __init__(self, value): @@ -118,7 +124,6 @@ class CacheRegionTest(test_cache.BaseTestCase): return self.test_value def _do_test(value): - test_obj = _test_obj(value) # Ensure the value has been cached @@ -130,19 +135,22 @@ class CacheRegionTest(test_cache.BaseTestCase): self.assertEqual(cached_value.value, test_obj.test_value.value) # Change the underlying value on the test object. test_obj.test_value = TestProxyValue( - uuidutils.generate_uuid(dashed=False)) - self.assertEqual(cached_value.value, - test_obj.get_test_value().value) + uuidutils.generate_uuid(dashed=False) + ) + self.assertEqual( + cached_value.value, test_obj.get_test_value().value + ) # override the system time to ensure the non-cached new value # is returned new_time = time.time() + (cache_time * 2) - with mock.patch.object(time, 'time', - return_value=new_time): + with mock.patch.object(time, 'time', return_value=new_time): overriden_cache_value = test_obj.get_test_value() - self.assertNotEqual(cached_value.value, - overriden_cache_value.value) - self.assertEqual(test_obj.test_value.value, - overriden_cache_value.value) + self.assertNotEqual( + cached_value.value, overriden_cache_value.value + ) + self.assertEqual( + test_obj.test_value.value, overriden_cache_value.value + ) return _do_test @@ -153,11 +161,11 @@ class CacheRegionTest(test_cache.BaseTestCase): # [cache] group cache_time = 600 expiration_time = cache._get_expiration_time_fn( - self.config_fixture.conf, TEST_GROUP) + self.config_fixture.conf, TEST_GROUP + ) do_test = self._get_cache_fallthrough_fn(cache_time) # Run the test with the dummy group cache_time value - self.config_fixture.config(cache_time=cache_time, - group=TEST_GROUP) + self.config_fixture.config(cache_time=cache_time, group=TEST_GROUP) test_value = TestProxyValue(uuidutils.generate_uuid(dashed=False)) self.assertEqual(cache_time, expiration_time()) do_test(value=test_value) @@ -169,13 +177,13 @@ class CacheRegionTest(test_cache.BaseTestCase): # [cache] group cache_time = 599 expiration_time = cache._get_expiration_time_fn( - self.config_fixture.conf, TEST_GROUP) + self.config_fixture.conf, TEST_GROUP + ) do_test = self._get_cache_fallthrough_fn(cache_time) # Run the test with the dummy group cache_time value set to None and # the global value set. self.config_fixture.config(cache_time=None, group=TEST_GROUP) - test_value = TestProxyValue( - uuidutils.generate_uuid(dashed=False)) + test_value = TestProxyValue(uuidutils.generate_uuid(dashed=False)) self.assertIsNone(expiration_time()) do_test(value=test_value) @@ -243,49 +251,56 @@ class CacheRegionTest(test_cache.BaseTestCase): def test_cache_config_builder(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='oslo_cache.dict', - expiration_time=86400, - backend_argument=['arg1:test', - 'arg2:test:test', - 'arg3.invalid']) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='oslo_cache.dict', + expiration_time=86400, + backend_argument=['arg1:test', 'arg2:test:test', 'arg3.invalid'], + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( self.config_fixture.conf.cache.backend, - config_dict['test_prefix.backend']) + config_dict['test_prefix.backend'], + ) self.assertEqual( self.config_fixture.conf.cache.expiration_time, - config_dict['test_prefix.expiration_time']) + config_dict['test_prefix.expiration_time'], + ) self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) - self.assertEqual('test:test', - config_dict['test_prefix.arguments.arg2']) + self.assertEqual( + 'test:test', config_dict['test_prefix.arguments.arg2'] + ) self.assertNotIn('test_prefix.arguments.arg3', config_dict) def test_cache_config_builder_global_disabled(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=False, - config_prefix='test_prefix', - backend='oslo_cache.dict') + self.config_fixture.config( + group='cache', + enabled=False, + config_prefix='test_prefix', + backend='oslo_cache.dict', + ) self.assertFalse(self.config_fixture.conf.cache.enabled) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - _opts._DEFAULT_BACKEND, - config_dict['test_prefix.backend']) + _opts._DEFAULT_BACKEND, config_dict['test_prefix.backend'] + ) def test_cache_config_builder_tls_disabled(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file', - tls_allowed_ciphers='allowed_ciphers') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + tls_allowed_ciphers='allowed_ciphers', + ) with mock.patch.object(ssl, 'create_default_context'): config_dict = cache._build_cache_config(self.config_fixture.conf) @@ -296,52 +311,61 @@ class CacheRegionTest(test_cache.BaseTestCase): def test_cache_config_builder_tls_disabled_redis(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis', - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file', - tls_allowed_ciphers='allowed_ciphers') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis', + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + tls_allowed_ciphers='allowed_ciphers', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( 'redis://localhost:6379/0', - config_dict['test_prefix.arguments.url']) + config_dict['test_prefix.arguments.url'], + ) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) - self.assertNotIn('test_prefix.arguments.connection_kwargs', - config_dict) + self.assertNotIn( + 'test_prefix.arguments.connection_kwargs', config_dict + ) def test_cache_config_builder_tls_disabled_redis_sentinel(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel', - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) - self.assertNotIn('test_prefix.arguments.connection_kwargs', - config_dict) - self.assertNotIn('test_prefix.arguments.sentinel_kwargs', - config_dict) + self.assertNotIn( + 'test_prefix.arguments.connection_kwargs', config_dict + ) + self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict) def test_cache_config_builder_tls_enabled(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - tls_enabled=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + tls_enabled=True, + ) fake_context = mock.Mock() - with mock.patch.object(ssl, 'create_default_context', - return_value=fake_context): + with mock.patch.object( + ssl, 'create_default_context', return_value=fake_context + ): config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertTrue(self.config_fixture.conf.cache.tls_enabled) @@ -357,148 +381,171 @@ class CacheRegionTest(test_cache.BaseTestCase): def test_cache_config_builder_tls_enabled_redis(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis', - tls_enabled=True, - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis', + tls_enabled=True, + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertTrue(self.config_fixture.conf.cache.tls_enabled) - self.assertIn('test_prefix.arguments.connection_kwargs', - config_dict) + self.assertIn('test_prefix.arguments.connection_kwargs', config_dict) self.assertEqual( 'rediss://localhost:6379/0', - config_dict['test_prefix.arguments.url']) + config_dict['test_prefix.arguments.url'], + ) self.assertEqual( { 'ssl_ca_certs': 'path_to_ca_file', 'ssl_keyfile': 'path_to_key_file', - 'ssl_certfile': 'path_to_cert_file' + 'ssl_certfile': 'path_to_cert_file', }, - config_dict['test_prefix.arguments.connection_kwargs']) + config_dict['test_prefix.arguments.connection_kwargs'], + ) self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict) def test_cache_config_builder_tls_enabled_redis_sentinel(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel', - tls_enabled=True, - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + tls_enabled=True, + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertTrue(self.config_fixture.conf.cache.tls_enabled) - self.assertIn('test_prefix.arguments.connection_kwargs', - config_dict) + self.assertIn('test_prefix.arguments.connection_kwargs', config_dict) self.assertEqual( { 'ssl': True, 'ssl_ca_certs': 'path_to_ca_file', 'ssl_keyfile': 'path_to_key_file', - 'ssl_certfile': 'path_to_cert_file' + 'ssl_certfile': 'path_to_cert_file', }, - config_dict['test_prefix.arguments.connection_kwargs']) - self.assertIn('test_prefix.arguments.sentinel_kwargs', - config_dict) + config_dict['test_prefix.arguments.connection_kwargs'], + ) + self.assertIn('test_prefix.arguments.sentinel_kwargs', config_dict) self.assertEqual( { 'ssl': True, 'ssl_ca_certs': 'path_to_ca_file', 'ssl_keyfile': 'path_to_key_file', - 'ssl_certfile': 'path_to_cert_file' + 'ssl_certfile': 'path_to_cert_file', }, - config_dict['test_prefix.arguments.sentinel_kwargs']) + config_dict['test_prefix.arguments.sentinel_kwargs'], + ) @mock.patch('oslo_cache.core._LOG') def test_cache_config_builder_fips_mode_supported(self, log): """Validate the FIPS mode is supported.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - tls_enabled=True, - enforce_fips_mode=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + tls_enabled=True, + enforce_fips_mode=True, + ) # Ensure that we emulate FIPS_mode even if it doesn't exist - with mock.patch.object(ssl, 'FIPS_mode', - create=True, return_value=True): + with mock.patch.object( + ssl, 'FIPS_mode', create=True, return_value=True + ): # Ensure that we are able to set FIPS_mode with mock.patch.object(ssl, 'FIPS_mode_set', create=True): - cache._build_cache_config(self.config_fixture.conf) log.info.assert_called_once_with( - "Enforcing the use of the OpenSSL FIPS mode") + "Enforcing the use of the OpenSSL FIPS mode" + ) @mock.patch('oslo_cache.core._LOG') def test_cache_config_builder_fips_mode_unsupported(self, log): """Validate the FIPS mode is not supported.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - tls_enabled=True, - enforce_fips_mode=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + tls_enabled=True, + enforce_fips_mode=True, + ) with mock.patch.object(cache, 'ssl') as ssl_: del ssl_.FIPS_mode # We do this test only if FIPS mode is not supported to # ensure that we hard fail. - self.assertRaises(exception.ConfigurationError, - cache._build_cache_config, - self.config_fixture.conf) + self.assertRaises( + exception.ConfigurationError, + cache._build_cache_config, + self.config_fixture.conf, + ) def test_cache_config_builder_fips_mode_unsupported_redis(self): """Validate the FIPS mode is not supported.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis', - tls_enabled=True, - enforce_fips_mode=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis', + tls_enabled=True, + enforce_fips_mode=True, + ) - self.assertRaises(exception.ConfigurationError, - cache._build_cache_config, - self.config_fixture.conf) + self.assertRaises( + exception.ConfigurationError, + cache._build_cache_config, + self.config_fixture.conf, + ) def test_cache_config_builder_tls_enabled_unsupported(self): """Validate the tls_enabled opiton is not supported..""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='oslo_cache.dict', - tls_enabled=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + tls_enabled=True, + ) with mock.patch.object(ssl, 'create_default_context'): - self.assertRaises(exception.ConfigurationError, - cache._build_cache_config, - self.config_fixture.conf) + self.assertRaises( + exception.ConfigurationError, + cache._build_cache_config, + self.config_fixture.conf, + ) ssl.create_default_context.assert_not_called() def test_cache_config_builder_tls_enabled_with_config(self): """Validate the backend is reset to default if caching is disabled.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - tls_enabled=True, - tls_cafile='path_to_ca_file', - tls_keyfile='path_to_key_file', - tls_certfile='path_to_cert_file', - tls_allowed_ciphers='allowed_ciphers') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + tls_enabled=True, + tls_cafile='path_to_ca_file', + tls_keyfile='path_to_key_file', + tls_certfile='path_to_cert_file', + tls_allowed_ciphers='allowed_ciphers', + ) fake_context = mock.Mock() - with mock.patch.object(ssl, 'create_default_context', - return_value=fake_context): + with mock.patch.object( + ssl, 'create_default_context', return_value=fake_context + ): config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertTrue(self.config_fixture.conf.cache.tls_enabled) @@ -510,9 +557,7 @@ class CacheRegionTest(test_cache.BaseTestCase): 'path_to_cert_file', 'path_to_key_file', ) - fake_context.set_ciphers.assert_called_with( - 'allowed_ciphers' - ) + fake_context.set_ciphers.assert_called_with('allowed_ciphers') self.assertEqual( fake_context, @@ -523,130 +568,138 @@ class CacheRegionTest(test_cache.BaseTestCase): """Validate we build a config without the retry option when retry is disabled. """ - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='oslo_cache.dict', - enable_socket_keepalive=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + enable_socket_keepalive=True, + ) self.assertRaises( exception.ConfigurationError, cache._build_cache_config, - self.config_fixture.conf + self.config_fixture.conf, ) def test_cache_pymemcache_socket_keepalive_disabled(self): """Validate we build a dogpile.cache dict config without keepalive.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - socket_keepalive_idle=2, - socket_keepalive_interval=2, - socket_keepalive_count=2) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + socket_keepalive_idle=2, + socket_keepalive_interval=2, + socket_keepalive_count=2, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse( - self.config_fixture.conf.cache.enable_socket_keepalive) - self.assertNotIn( - 'test_prefix.arguments.socket_keepalive', config_dict) + self.config_fixture.conf.cache.enable_socket_keepalive + ) + self.assertNotIn('test_prefix.arguments.socket_keepalive', config_dict) def test_cache_pymemcache_socket_keepalive_enabled(self): """Validate we build a dogpile.cache dict config with keepalive.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - enable_socket_keepalive=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_socket_keepalive=True, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertTrue( - self.config_fixture.conf.cache.enable_socket_keepalive) + self.assertTrue(self.config_fixture.conf.cache.enable_socket_keepalive) self.assertIsInstance( config_dict['test_prefix.arguments.socket_keepalive'], - KeepaliveOpts + KeepaliveOpts, ) def test_cache_pymemcache_socket_keepalive_with_config(self): """Validate we build a socket keepalive with the right config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - enable_socket_keepalive=True, - socket_keepalive_idle=12, - socket_keepalive_interval=38, - socket_keepalive_count=42) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_socket_keepalive=True, + socket_keepalive_idle=12, + socket_keepalive_interval=38, + socket_keepalive_count=42, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertTrue( - self.config_fixture.conf.cache.enable_socket_keepalive) + self.assertTrue(self.config_fixture.conf.cache.enable_socket_keepalive) self.assertTrue( config_dict['test_prefix.arguments.socket_keepalive'], - KeepaliveOpts + KeepaliveOpts, ) self.assertEqual( - 12, - config_dict['test_prefix.arguments.socket_keepalive'].idle + 12, config_dict['test_prefix.arguments.socket_keepalive'].idle ) self.assertEqual( - 38, - config_dict['test_prefix.arguments.socket_keepalive'].intvl + 38, config_dict['test_prefix.arguments.socket_keepalive'].intvl ) self.assertEqual( - 42, - config_dict['test_prefix.arguments.socket_keepalive'].cnt + 42, config_dict['test_prefix.arguments.socket_keepalive'].cnt ) def test_cache_pymemcache_retry_enabled_with_wrong_backend(self): """Validate we build a config without the retry option when retry is disabled. """ - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='oslo_cache.dict', - enable_retry_client=True, - retry_attempts=2, - retry_delay=2) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + enable_retry_client=True, + retry_attempts=2, + retry_delay=2, + ) self.assertRaises( exception.ConfigurationError, cache._build_cache_config, - self.config_fixture.conf + self.config_fixture.conf, ) def test_cache_pymemcache_retry_disabled(self): """Validate we build a config without the retry option when retry is disabled. """ - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - retry_attempts=2, - retry_delay=2) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + retry_attempts=2, + retry_delay=2, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) opts = ['enable_retry_client', 'retry_attempts', 'retry_delay'] for el in opts: - self.assertNotIn(f'test_prefix.arguments.{el}', - config_dict) + self.assertNotIn(f'test_prefix.arguments.{el}', config_dict) def test_cache_pymemcache_retry_enabled(self): """Validate we build a dogpile.cache dict config with retry.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - enable_retry_client=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_retry_client=True, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) @@ -657,362 +710,416 @@ class CacheRegionTest(test_cache.BaseTestCase): def test_cache_pymemcache_retry_with_opts(self): """Validate we build a valid config for the retry client.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - enable_retry_client=True, - retry_attempts=42, - retry_delay=42) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_retry_client=True, + retry_attempts=42, + retry_delay=42, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertTrue( - self.config_fixture.conf.cache.enable_retry_client) + self.assertTrue(self.config_fixture.conf.cache.enable_retry_client) self.assertEqual( - config_dict['test_prefix.arguments.retry_attempts'], - 42 + config_dict['test_prefix.arguments.retry_attempts'], 42 ) - self.assertEqual( - config_dict['test_prefix.arguments.retry_delay'], - 42 - ) + self.assertEqual(config_dict['test_prefix.arguments.retry_delay'], 42) def test_cache_pymemcache_retry_with_extra_opts(self): """Validate we build a valid config for the retry client.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.pymemcache', - enable_retry_client=True, - retry_attempts=42, - retry_delay=42, - hashclient_retry_attempts=100, - hashclient_retry_delay=100, - dead_timeout=100) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_retry_client=True, + retry_attempts=42, + retry_delay=42, + hashclient_retry_attempts=100, + hashclient_retry_delay=100, + dead_timeout=100, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertTrue( - self.config_fixture.conf.cache.enable_retry_client) + self.assertTrue(self.config_fixture.conf.cache.enable_retry_client) self.assertEqual( - config_dict['test_prefix.arguments.retry_attempts'], - 42 + config_dict['test_prefix.arguments.retry_attempts'], 42 + ) + + self.assertEqual(config_dict['test_prefix.arguments.retry_delay'], 42) + + self.assertEqual( + config_dict['test_prefix.arguments.hashclient_retry_attempts'], 100 ) self.assertEqual( - config_dict['test_prefix.arguments.retry_delay'], - 42 + config_dict['test_prefix.arguments.hashclient_retry_delay'], 100 ) self.assertEqual( - config_dict['test_prefix.arguments.hashclient_retry_attempts'], - 100 - ) - - self.assertEqual( - config_dict['test_prefix.arguments.hashclient_retry_delay'], - 100 - ) - - self.assertEqual( - config_dict['test_prefix.arguments.dead_timeout'], - 100 + config_dict['test_prefix.arguments.dead_timeout'], 100 ) def test_cache_config_builder_flush_on_reconnect_enabled(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='oslo_cache.dict', - memcache_pool_flush_on_reconnect=True) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + memcache_pool_flush_on_reconnect=True, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertTrue(self.config_fixture.conf.cache. - memcache_pool_flush_on_reconnect) - self.assertTrue(config_dict['test_prefix.arguments' - '.pool_flush_on_reconnect']) + self.assertTrue( + self.config_fixture.conf.cache.memcache_pool_flush_on_reconnect + ) + self.assertTrue( + config_dict['test_prefix.arguments.pool_flush_on_reconnect'] + ) def test_cache_config_builder_flush_on_reconnect_disabled(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='oslo_cache.dict', - memcache_pool_flush_on_reconnect=False) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + memcache_pool_flush_on_reconnect=False, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) - self.assertFalse(self.config_fixture.conf.cache. - memcache_pool_flush_on_reconnect) - self.assertFalse(config_dict['test_prefix.arguments' - '.pool_flush_on_reconnect']) + self.assertFalse( + self.config_fixture.conf.cache.memcache_pool_flush_on_reconnect + ) + self.assertFalse( + config_dict['test_prefix.arguments.pool_flush_on_reconnect'] + ) def test_cache_config_builder_redis(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379') + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - 'redis://[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + 'redis://[::1]:6379/0', config_dict['test_prefix.arguments.url'] + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) def test_cache_config_builder_redis_with_db(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - redis_db=1) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_db=1, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - 'redis://[::1]:6379/1', - config_dict['test_prefix.arguments.url']) + 'redis://[::1]:6379/1', config_dict['test_prefix.arguments.url'] + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) def test_cache_config_builder_redis_with_sock_to(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - redis_socket_timeout=10.0) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_socket_timeout=10.0, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - 'redis://[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + 'redis://[::1]:6379/0', config_dict['test_prefix.arguments.url'] + ) self.assertEqual( - 10.0, config_dict['test_prefix.arguments.socket_timeout']) + 10.0, config_dict['test_prefix.arguments.socket_timeout'] + ) def test_cache_config_builder_redis_with_keepalive(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - enable_socket_keepalive=True) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + enable_socket_keepalive=True, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - 'redis://[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + 'redis://[::1]:6379/0', config_dict['test_prefix.arguments.url'] + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) - self.assertEqual({ - 'socket_keepalive': True, - 'socket_keepalive_options': { - socket.TCP_KEEPIDLE: 1, - socket.TCP_KEEPINTVL: 1, - socket.TCP_KEEPCNT: 1, - }}, config_dict['test_prefix.arguments.connection_kwargs']) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) + self.assertEqual( + { + 'socket_keepalive': True, + 'socket_keepalive_options': { + socket.TCP_KEEPIDLE: 1, + socket.TCP_KEEPINTVL: 1, + socket.TCP_KEEPCNT: 1, + }, + }, + config_dict['test_prefix.arguments.connection_kwargs'], + ) def test_cache_config_builder_redis_with_keepalive_params(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - enable_socket_keepalive=True, - socket_keepalive_idle=2, - socket_keepalive_interval=3, - socket_keepalive_count=4) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + enable_socket_keepalive=True, + socket_keepalive_idle=2, + socket_keepalive_interval=3, + socket_keepalive_count=4, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( - 'redis://[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + 'redis://[::1]:6379/0', config_dict['test_prefix.arguments.url'] + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) - self.assertEqual({ - 'socket_keepalive': True, - 'socket_keepalive_options': { - socket.TCP_KEEPIDLE: 2, - socket.TCP_KEEPINTVL: 3, - socket.TCP_KEEPCNT: 4, - }}, config_dict['test_prefix.arguments.connection_kwargs']) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) + self.assertEqual( + { + 'socket_keepalive': True, + 'socket_keepalive_options': { + socket.TCP_KEEPIDLE: 2, + socket.TCP_KEEPINTVL: 3, + socket.TCP_KEEPCNT: 4, + }, + }, + config_dict['test_prefix.arguments.connection_kwargs'], + ) def test_cache_config_builder_redis_with_auth(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - redis_password='secrete') + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_password='secrete', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( 'redis://:secrete@[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + config_dict['test_prefix.arguments.url'], + ) def test_cache_config_builder_redis_with_auth_and_user(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - redis_server='[::1]:6379', - redis_username='user', - redis_password='secrete') + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_username='user', + redis_password='secrete', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertEqual( 'redis://user:secrete@[::1]:6379/0', - config_dict['test_prefix.arguments.url']) + config_dict['test_prefix.arguments.url'], + ) def test_cache_config_builder_redis_sentinel(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) + self.assertEqual(0, config_dict['test_prefix.arguments.db']) self.assertEqual( - 0, config_dict['test_prefix.arguments.db']) + 'mymaster', config_dict['test_prefix.arguments.service_name'] + ) self.assertEqual( - 'mymaster', config_dict['test_prefix.arguments.service_name']) - self.assertEqual([ - ('localhost', 26379) - ], config_dict['test_prefix.arguments.sentinels']) + [('localhost', 26379)], + config_dict['test_prefix.arguments.sentinels'], + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) - self.assertNotIn('test_prefix.arguments.connection_kwargs', - config_dict) - self.assertNotIn('test_prefix.arguments.sentinel_kwargs', - config_dict) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) + self.assertNotIn( + 'test_prefix.arguments.connection_kwargs', config_dict + ) + self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict) def test_cache_config_builder_redis_sentinel_with_db(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel', - redis_db=1) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + redis_db=1, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) + self.assertEqual(1, config_dict['test_prefix.arguments.db']) self.assertEqual( - 1, config_dict['test_prefix.arguments.db']) + 'mymaster', config_dict['test_prefix.arguments.service_name'] + ) self.assertEqual( - 'mymaster', config_dict['test_prefix.arguments.service_name']) - self.assertEqual([ - ('localhost', 26379) - ], config_dict['test_prefix.arguments.sentinels']) + [('localhost', 26379)], + config_dict['test_prefix.arguments.sentinels'], + ) self.assertEqual( - 1.0, config_dict['test_prefix.arguments.socket_timeout']) - self.assertNotIn('test_prefix.arguments.connection_kwargs', - config_dict) - self.assertNotIn('test_prefix.arguments.sentinel_kwargs', - config_dict) + 1.0, config_dict['test_prefix.arguments.socket_timeout'] + ) + self.assertNotIn( + 'test_prefix.arguments.connection_kwargs', config_dict + ) + self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict) def test_cache_config_builder_redis_sentinel_with_sock_to(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel', - redis_socket_timeout=10.0) + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + redis_socket_timeout=10.0, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) + self.assertEqual(0, config_dict['test_prefix.arguments.db']) self.assertEqual( - 0, config_dict['test_prefix.arguments.db']) + 'mymaster', config_dict['test_prefix.arguments.service_name'] + ) self.assertEqual( - 'mymaster', config_dict['test_prefix.arguments.service_name']) - self.assertEqual([ - ('localhost', 26379) - ], config_dict['test_prefix.arguments.sentinels']) + [('localhost', 26379)], + config_dict['test_prefix.arguments.sentinels'], + ) self.assertEqual( - 10.0, config_dict['test_prefix.arguments.socket_timeout']) - self.assertNotIn('test_prefix.arguments.connection_kwargs', - config_dict) - self.assertNotIn('test_prefix.arguments.sentinel_kwargs', - config_dict) + 10.0, config_dict['test_prefix.arguments.socket_timeout'] + ) + self.assertNotIn( + 'test_prefix.arguments.connection_kwargs', config_dict + ) + self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict) def test_cache_config_builder_redis_sentinel_with_auth(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - enabled=True, - config_prefix='test_prefix', - backend='dogpile.cache.redis_sentinel', - redis_username='user', - redis_password='secrete', - redis_db=1, - redis_sentinels=[ - '127.0.0.1:26379', - '[::1]:26379', - 'localhost:26379' - ], - redis_sentinel_service_name='cluster') + self.config_fixture.config( + group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.redis_sentinel', + redis_username='user', + redis_password='secrete', + redis_db=1, + redis_sentinels=[ + '127.0.0.1:26379', + '[::1]:26379', + 'localhost:26379', + ], + redis_sentinel_service_name='cluster', + ) config_dict = cache._build_cache_config(self.config_fixture.conf) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) + self.assertEqual(1, config_dict['test_prefix.arguments.db']) self.assertEqual( - 1, config_dict['test_prefix.arguments.db']) + 'cluster', config_dict['test_prefix.arguments.service_name'] + ) self.assertEqual( - 'cluster', config_dict['test_prefix.arguments.service_name']) - self.assertEqual([ - ('127.0.0.1', 26379), - ('::1', 26379), - ('localhost', 26379), - ], config_dict['test_prefix.arguments.sentinels']) + [ + ('127.0.0.1', 26379), + ('::1', 26379), + ('localhost', 26379), + ], + config_dict['test_prefix.arguments.sentinels'], + ) + self.assertEqual('user', config_dict['test_prefix.arguments.username']) self.assertEqual( - 'user', config_dict['test_prefix.arguments.username']) - self.assertEqual( - 'secrete', config_dict['test_prefix.arguments.password']) + 'secrete', config_dict['test_prefix.arguments.password'] + ) def test_cache_config_builder_with_backend_expiration(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.memcached', - backend_expiration_time=600) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.memcached', + backend_expiration_time=600, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual(600, config_dict['test_prefix.expiration_time']) self.assertEqual( - 600, config_dict['test_prefix.expiration_time']) - self.assertEqual( - 600, config_dict['test_prefix.arguments.memcached_expire_time']) + 600, config_dict['test_prefix.arguments.memcached_expire_time'] + ) def test_cache_config_builder_with_redis_backend_expiration(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.redis', - backend_expiration_time=600) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + backend_expiration_time=600, + ) config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual(600, config_dict['test_prefix.expiration_time']) self.assertEqual( - 600, config_dict['test_prefix.expiration_time']) - self.assertEqual( - 600, config_dict['test_prefix.arguments.redis_expiration_time']) + 600, config_dict['test_prefix.arguments.redis_expiration_time'] + ) def test_cache_config_builder_with_backend_expiration_too_small(self): """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='dogpile.cache.memcached', - backend_expiration_time=599) + self.config_fixture.config( + group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.memcached', + backend_expiration_time=599, + ) - self.assertRaises(exception.ConfigurationError, - cache._build_cache_config, self.config_fixture.conf) + self.assertRaises( + exception.ConfigurationError, + cache._build_cache_config, + self.config_fixture.conf, + ) def test_cache_debug_proxy(self): single_value = 'Test Value' @@ -1036,14 +1143,17 @@ class CacheRegionTest(test_cache.BaseTestCase): self.assertEqual(NO_VALUE, value) def test_configure_non_region_object_raises_error(self): - self.assertRaises(exception.ConfigurationError, - cache.configure_cache_region, - self.config_fixture.conf, - "bogus") + self.assertRaises( + exception.ConfigurationError, + cache.configure_cache_region, + self.config_fixture.conf, + "bogus", + ) def test_kwarg_function_key_generator_no_kwargs(self): cacheable_function = self._get_cacheable_function( - region=self.region_kwargs) + region=self.region_kwargs + ) self.config_fixture.config(group='cache', enabled=True) cacheable_function(self.test_value) @@ -1052,7 +1162,8 @@ class CacheRegionTest(test_cache.BaseTestCase): def test_kwarg_function_key_generator_with_kwargs(self): cacheable_function = self._get_cacheable_function( - region=self.region_kwargs) + region=self.region_kwargs + ) self.config_fixture.config(group='cache', enabled=True) cacheable_function(value=self.test_value) @@ -1061,7 +1172,6 @@ class CacheRegionTest(test_cache.BaseTestCase): class UTF8KeyManglerTests(test_cache.BaseTestCase): - def test_key_is_utf8_encoded(self): key = 'fäké1' encoded = cache._sha1_mangle_key(key) diff --git a/oslo_cache/tests/unit/test_connection_pool.py b/oslo_cache/tests/unit/test_connection_pool.py index 5ee89100..42729fa7 100644 --- a/oslo_cache/tests/unit/test_connection_pool.py +++ b/oslo_cache/tests/unit/test_connection_pool.py @@ -40,8 +40,8 @@ class TestConnectionPool(test_cache.BaseTestCase): self.unused_timeout = 10 self.maxsize = 2 self.connection_pool = _TestConnectionPool( - maxsize=self.maxsize, - unused_timeout=self.unused_timeout) + maxsize=self.maxsize, unused_timeout=self.unused_timeout + ) self.addCleanup(self.cleanup_instance('connection_pool')) def cleanup_instance(self, *names): @@ -54,6 +54,7 @@ class TestConnectionPool(test_cache.BaseTestCase): for name in names: if hasattr(self, name): delattr(self, name) + return cleanup def test_get_context_manager(self): @@ -68,8 +69,8 @@ class TestConnectionPool(test_cache.BaseTestCase): self.test_get_context_manager() newtime = time.time() + self.unused_timeout * 2 non_expired_connection = _memcache_pool._PoolItem( - ttl=(newtime * 2), - connection=mock.MagicMock()) + ttl=(newtime * 2), connection=mock.MagicMock() + ) self.connection_pool.queue.append(non_expired_connection) self.assertThat(self.connection_pool.queue, matchers.HasLength(2)) with mock.patch.object(time, 'time', return_value=newtime): @@ -77,7 +78,8 @@ class TestConnectionPool(test_cache.BaseTestCase): with self.connection_pool.acquire(): pass conn.assert_has_calls( - [mock.call(self.connection_pool.destroyed_value)]) + [mock.call(self.connection_pool.destroyed_value)] + ) self.assertThat(self.connection_pool.queue, matchers.HasLength(1)) self.assertEqual(0, non_expired_connection.connection.call_count) @@ -85,13 +87,15 @@ class TestConnectionPool(test_cache.BaseTestCase): class TestException(Exception): pass - with mock.patch.object(_TestConnectionPool, '_create_connection', - side_effect=TestException): + with mock.patch.object( + _TestConnectionPool, + '_create_connection', + side_effect=TestException, + ): with testtools.ExpectedException(TestException): with self.connection_pool.acquire(): pass - self.assertThat(self.connection_pool.queue, - matchers.HasLength(0)) + self.assertThat(self.connection_pool.queue, matchers.HasLength(0)) self.assertEqual(0, self.connection_pool._acquired) def test_connection_pool_limits_maximum_connections(self): @@ -114,9 +118,8 @@ class TestConnectionPool(test_cache.BaseTestCase): def test_connection_pool_maximum_connection_get_timeout(self): connection_pool = _TestConnectionPool( - maxsize=1, - unused_timeout=self.unused_timeout, - conn_get_timeout=0) + maxsize=1, unused_timeout=self.unused_timeout, conn_get_timeout=0 + ) def _acquire_connection(): with connection_pool.acquire(): @@ -134,7 +137,6 @@ class TestConnectionPool(test_cache.BaseTestCase): class TestMemcacheClientOverrides(test_cache.BaseTestCase): - def test_client_stripped_of_threading_local(self): """threading.local overrides are restored for _MemcacheClient""" client_class = _memcache_pool._MemcacheClient @@ -143,8 +145,10 @@ class TestMemcacheClientOverrides(test_cache.BaseTestCase): self.assertTrue(thread_local is threading.local) for field in thread_local.__dict__.keys(): if field not in ('__dict__', '__weakref__'): - self.assertNotEqual(id(getattr(thread_local, field, None)), - id(getattr(client_class, field, None))) + self.assertNotEqual( + id(getattr(thread_local, field, None)), + id(getattr(client_class, field, None)), + ) def test_can_create_with_kwargs(self): """Test for lp 1812935 @@ -153,6 +157,7 @@ class TestMemcacheClientOverrides(test_cache.BaseTestCase): following to the top of oslo_cache/tests/__init__.py:: import eventlet + eventlet.monkey_patch() This should happen before any other imports in that file. @@ -165,7 +170,6 @@ class TestMemcacheClientOverrides(test_cache.BaseTestCase): class TestBMemcacheClient(test_cache.BaseTestCase): - def test_can_create_with_kwargs(self): client = _bmemcache_pool._BMemcacheClient('foo', password='123456') # Make sure kwargs are properly processed by the client diff --git a/oslo_cache/tests/unit/test_dict_backend.py b/oslo_cache/tests/unit/test_dict_backend.py index 1d39f27e..154282aa 100644 --- a/oslo_cache/tests/unit/test_dict_backend.py +++ b/oslo_cache/tests/unit/test_dict_backend.py @@ -26,7 +26,6 @@ VALUE = 'test_value' class CacheDictBackendTest(test_cache.BaseTestCase): - def setUp(self): super().setUp() self.config_fixture = self.useFixture(config_fixture.Config()) @@ -34,7 +33,8 @@ class CacheDictBackendTest(test_cache.BaseTestCase): self.time_fixture = self.useFixture(time_fixture.TimeFixture()) self.region = dp_region.make_region() self.region.configure( - 'oslo_cache.dict', arguments={'expiration_time': 0.5}) + 'oslo_cache.dict', arguments={'expiration_time': 0.5} + ) def test_dict_backend(self): self.assertIs(NO_VALUE, self.region.get(KEY)) @@ -64,7 +64,8 @@ class CacheDictBackendTest(test_cache.BaseTestCase): def test_dict_backend_zero_expiration_time(self): self.region = dp_region.make_region() self.region.configure( - 'oslo_cache.dict', arguments={'expiration_time': 0}) + 'oslo_cache.dict', arguments={'expiration_time': 0} + ) self.region.set(KEY, VALUE) self.time_fixture.advance_time_seconds(1) diff --git a/pyproject.toml b/pyproject.toml index 52980def..e2368509 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,3 +53,16 @@ etcd3gw = [ packages = [ "oslo_cache" ] + +[tool.ruff] +line-length = 79 + +[tool.ruff.format] +quote-style = "preserve" +docstring-code-format = true + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "S", "U"] + +[tool.ruff.lint.per-file-ignores] +"oslo_cache/tests/*" = ["S"] diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index cc0ef970..da96ca40 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -194,10 +194,8 @@ htmlhelp_basename = 'oslo.cacheReleaseNotesDoc' latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # 'preamble': '', } @@ -206,9 +204,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'oslo.cacheReleaseNotes.tex', - 'oslo.cache Release Notes Documentation', - 'oslo.cache Developers', 'manual'), + ( + 'index', + 'oslo.cacheReleaseNotes.tex', + 'oslo.cache Release Notes Documentation', + 'oslo.cache Developers', + 'manual', + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -236,9 +238,13 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'oslo.cacheReleaseNotes', - 'oslo.cache Release Notes Documentation', - ['oslo.cache Developers'], 1) + ( + 'index', + 'oslo.cacheReleaseNotes', + 'oslo.cache Release Notes Documentation', + ['oslo.cache Developers'], + 1, + ) ] # If true, show URL addresses after external links. @@ -250,11 +256,15 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'oslo.cacheReleaseNotes', - 'oslo.cache Release Notes Documentation', - 'oslo.cache Developers', 'oslo.cacheReleaseNotes', - 'One line description of project.', - 'Miscellaneous'), + ( + 'index', + 'oslo.cacheReleaseNotes', + 'oslo.cache Release Notes Documentation', + 'oslo.cache Developers', + 'oslo.cacheReleaseNotes', + 'One line description of project.', + 'Miscellaneous', + ), ] # Documents to append as an appendix to all manuals. diff --git a/setup.py b/setup.py index cd35c3c3..481505b0 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,4 @@ import setuptools -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True) +setuptools.setup(setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/tox.ini b/tox.ini index 9c143c54..7edcfbe8 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,8 @@ commands = commands = {posargs} [testenv:docs] -allowlist_externals = rm +allowlist_externals = + rm deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt @@ -39,6 +40,14 @@ commands = rm -fr doc/build sphinx-build -W --keep-going -b html doc/source doc/build/html +[testenv:releasenotes] +allowlist_externals = + rm +deps = {[testenv:docs]deps} +commands = + rm -rf releasenotes/build + sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html + [testenv:cover] setenv = PYTHON=coverage run --source oslo_cache --parallel-mode @@ -51,18 +60,13 @@ commands = coverage report --show-missing [flake8] -show-source = True -ignore = H405,W504,F405 -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +show-source = true +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,__init__.py +# We only enable the hacking (H) checks +select = H +# H301 Black will put commas after imports that can't fit on one line +# H405 Multi-line docstrings are fine +ignore = H301,H405 [hacking] -import_exceptions = - -[testenv:releasenotes] -allowlist_externals = - rm -deps = {[testenv:docs]deps} -commands = - rm -rf releasenotes/build - sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html +import_exceptions = oslo_cache._i18n