From 130ada151a1e2e79f799f1df0ea0585f03b2309e Mon Sep 17 00:00:00 2001 From: kairat_kushaev Date: Wed, 15 Jul 2015 16:11:44 +0300 Subject: [PATCH] Enable caching for property constraints The patch adds support of oslo.caching in heat and enables caching when heat requests OS services and validates custom constraints. So it allows to increase performance of validation (cause we don't need to call requests for the same values) and handle the cases when stack have a lot of instances (and glance fails because heat is ddosing it). So the logic introduced in this patch is the following: - heat request value from validation cache - if value is found and it is not expired then we return it otherwise we need request client plugin about validation - if value is valid then store result in cache otherwise do not store it. The patch also introduces list of configurations that needs to be specified when using caching. The configuration for caching is modular. It consists from global toggle for caching (disabled by default because in-memory cache is leaking and memcached is not installed by default) and local toggles for specific piece of functionality (constraints cache). Local toggle is enabled by default but it won't be activated without global toggle(enabled=True in heat.conf). DocImpact Implements: blueprint constraint-validation-cache Change-Id: I0a71c52a5c003f0fc6be1762647ffe0146a3b387 --- heat/common/cache.py | 71 ++++++++++++++++++++++++++ heat/common/cache/__init__.py | 0 heat/common/cache/backends/__init__.py | 0 heat/common/cache/backends/noop.py | 48 ----------------- heat/engine/constraints.py | 48 ++++++++++++++--- requirements.txt | 1 + 6 files changed, 113 insertions(+), 55 deletions(-) create mode 100644 heat/common/cache.py delete mode 100644 heat/common/cache/__init__.py delete mode 100644 heat/common/cache/backends/__init__.py delete mode 100644 heat/common/cache/backends/noop.py diff --git a/heat/common/cache.py b/heat/common/cache.py new file mode 100644 index 0000000000..56d16fba7c --- /dev/null +++ b/heat/common/cache.py @@ -0,0 +1,71 @@ +# +# Copyright 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_cache import core +from oslo_config import cfg + +from heat.common.i18n import _ + +"""The module contains the code related to integration between oslo.cache +module and heat.""" + + +def register_cache_configurations(conf): + """Register all configurations required for oslo.cache + + The procedure registers all configurations required for oslo.cache. + It should be called before configuring of cache region + :param conf: instance of heat configuration + :return updated heat configuration + """ + # register global configurations for caching in heat + core.configure(conf) + + # register heat specific configurations + constraint_cache_group = cfg.OptGroup('constraint_validation_cache') + constraint_cache_opts = [ + cfg.IntOpt('expiration_time', default=60, + help=_( + 'TTL, in seconds, for any cached item in the ' + 'dogpile.cache region used for caching of validation ' + 'constraints.')), + cfg.BoolOpt("caching", default=True, + help=_( + 'Toggle to enable/disable caching when Orchestration ' + 'Engine validates property constraints of stack.' + 'During property validation with constraints ' + 'Orchestration Engine caches requests to other ' + 'OpenStack services. Please note that the global ' + 'toggle for oslo.cache(enabled=True in [cache] group) ' + 'must be enabled to use this feature.')) + ] + conf.register_group(constraint_cache_group) + conf.register_opts(constraint_cache_opts, group=constraint_cache_group) + + return conf + + +# variable that stores an initialized cache region for heat +_REGION = None + + +def get_cache_region(): + global _REGION + if not _REGION: + _REGION = core.configure_cache_region( + conf=register_cache_configurations(cfg.CONF), + region=core.create_region()) + return _REGION diff --git a/heat/common/cache/__init__.py b/heat/common/cache/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/heat/common/cache/backends/__init__.py b/heat/common/cache/backends/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/heat/common/cache/backends/noop.py b/heat/common/cache/backends/noop.py deleted file mode 100644 index 5046da5b7c..0000000000 --- a/heat/common/cache/backends/noop.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from dogpile.cache import api - - -NO_VALUE = api.NO_VALUE - - -class NoopCacheBackend(api.CacheBackend): - """A no op backend as a default caching backend. - - The no op backend is provided as the default caching backend for heat - to ensure that ``dogpile.cache.memory`` is not used in any real-world - circumstances unintentionally. ``dogpile.cache.memory`` does not have a - mechanism to cleanup it's internal dict and therefore could cause run-away - memory utilization. - """ - def __init__(self, *args): - return - - def get(self, key): - return NO_VALUE - - def get_multi(self, keys): - return [NO_VALUE for x in keys] - - def set(self, key, value): - return - - def set_multi(self, mapping): - return - - def delete(self, key): - return - - def delete_multi(self, keys): - return diff --git a/heat/engine/constraints.py b/heat/engine/constraints.py index e4dd709315..5861632c69 100644 --- a/heat/engine/constraints.py +++ b/heat/engine/constraints.py @@ -16,13 +16,22 @@ import numbers import re import warnings +from oslo_cache import core +from oslo_config import cfg from oslo_utils import strutils import six +from heat.common import cache from heat.common import exception from heat.common.i18n import _ from heat.engine import resources +# decorator that allows to cache the value +# of the function based on input arguments +MEMOIZE = core.get_memoization_decorator(conf=cfg.CONF, + region=cache.get_cache_region(), + group="constraint_validation_cache") + class Schema(collections.Mapping): """ @@ -582,10 +591,35 @@ class BaseCustomConstraint(object): "value": value, "message": self._error_message} def validate(self, value, context): - try: - self.validate_with_client(context.clients, value) - except self.expected_exceptions as e: - self._error_message = str(e) - return False - else: - return True + + @MEMOIZE + def check_cache_or_validate_value(class_name, value_to_validate): + """Check if validation result stored in cache or validate value. + + The function checks that value was validated and validation + result stored in cache. If not then it executes validation and + stores the result of validation in cache. + If caching is disable it requests for validation each time. + + :param class_name: name of custom constraint class, it will be + used in cache key to distinguish values + :param value_to_validate: value that need to be validated + :return: True if value is valid otherwise False + """ + try: + self.validate_with_client(context.clients, value_to_validate) + except self.expected_exceptions as e: + self._error_message = str(e) + return False + else: + return True + + validation_result = check_cache_or_validate_value( + self.__class__.__name__, value) + # if validation failed we should not store it in cache + # cause validation will be fixed soon (by admin or other guy) + # and we don't need to require user wait for expiration time + if not validation_result: + check_cache_or_validate_value.invalidate(self.__class__.__name__, + value) + return validation_result diff --git a/requirements.txt b/requirements.txt index 5ed1ab9daf..3b314f48ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ keystonemiddleware>=2.0.0 kombu>=3.0.7 lxml>=2.3 netaddr>=0.7.12 +oslo.cache>=0.3.0 # Apache-2.0 oslo.config>=1.11.0 # Apache-2.0 oslo.concurrency>=2.1.0 # Apache-2.0 oslo.context>=0.2.0 # Apache-2.0