diff --git a/lower-constraints.txt b/lower-constraints.txt index cf231e5..8a1527b 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -5,6 +5,7 @@ coverage==4.0 debtcollector==1.2.0 docutils==0.11 dulwich==0.15.0 +eventlet==0.20.0 extras==1.0.0 fixtures==3.0.0 flake8==2.5.5 @@ -25,6 +26,7 @@ netifaces==0.10.4 openstackdocstheme==1.18.1 os-client-config==1.28.0 oslo.i18n==3.15.3 +oslo.log==3.36.0 oslo.serialization==2.18.0 oslo.utils==3.33.0 oslotest==3.2.0 diff --git a/mistral_lib/tests/test_utils.py b/mistral_lib/tests/test_utils.py index 599aaac..0cba20b 100644 --- a/mistral_lib/tests/test_utils.py +++ b/mistral_lib/tests/test_utils.py @@ -14,48 +14,176 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import copy + from mistral_lib.tests import base as tests_base from mistral_lib import utils +import testtools.matchers as ttm + + +LEFT = { + 'key1': { + 'key11': "val11" + }, + 'key2': 'val2' +} + +RIGHT = { + 'key1': { + 'key11': "val111111", + 'key12': "val12", + 'key13': { + 'key131': 'val131' + } + }, + 'key2': 'val2222', + 'key3': 'val3' +} class TestUtils(tests_base.TestCase): + def test_merge_dicts(self): + left = copy.deepcopy(LEFT) + right = copy.deepcopy(RIGHT) + + expected = { + 'key1': { + 'key11': "val111111", + 'key12': "val12", + 'key13': { + 'key131': 'val131' + } + }, + 'key2': 'val2222', + 'key3': 'val3' + } + + utils.merge_dicts(left, right) + + self.assertDictEqual(left, expected) + + def test_merge_dicts_overwrite_false(self): + left = copy.deepcopy(LEFT) + right = copy.deepcopy(RIGHT) + + expected = { + 'key1': { + 'key11': "val11", + 'key12': "val12", + 'key13': { + 'key131': 'val131' + } + }, + 'key2': 'val2', + 'key3': 'val3' + } + + utils.merge_dicts(left, right, overwrite=False) + + self.assertDictEqual(left, expected) + + def test_itersubclasses(self): + class A(object): + pass + + class B(A): + pass + + class C(A): + pass + + class D(C): + pass + + self.assertEqual([B, C, D], list(utils.iter_subclasses(A))) + + def test_get_dict_from_entries(self): + input = ['param1', {'param2': 2}] + input_dict = utils.get_dict_from_entries(input) + + self.assertIn('param1', input_dict) + self.assertIn('param2', input_dict) + self.assertEqual(2, input_dict.get('param2')) + self.assertIs(input_dict.get('param1'), utils.NotDefined) + + def test_get_input_dict_from_string(self): + self.assertDictEqual( + { + 'param1': utils.NotDefined, + 'param2': 2, + 'param3': 'var3' + }, + utils.get_dict_from_string('param1, param2=2, param3="var3"') + ) + + self.assertDictEqual({}, utils.get_dict_from_string('')) + def test_cut_string(self): s = 'Hello, Mistral!' self.assertEqual('Hello...', utils.cut_string(s, length=5)) self.assertEqual(s, utils.cut_string(s, length=100)) + self.assertEqual(s, utils.cut_string(s, length=-1)) def test_cut_list(self): - l = ['Hello, Mistral!', 'Hello, OpenStack!'] + list_ = ['Hello, Mistral!', 'Hello, OpenStack!'] - self.assertEqual("['Hello, M...", utils.cut_list(l, 11)) - self.assertEqual("['Hello, Mistr...", utils.cut_list(l, 15)) - self.assertEqual("['Hello, Mistral!', 'He...", utils.cut_list(l, 24)) + self.assertEqual("['Hello, M...", utils.cut_list(list_, 13)) + self.assertEqual("['Hello, Mistr...", utils.cut_list(list_, 17)) + self.assertEqual( + "['Hello, Mistral!', 'He...", + utils.cut_list(list_, 26)) self.assertEqual( "['Hello, Mistral!', 'Hello, OpenStack!']", - utils.cut_list(l, 100) + utils.cut_list(list_, 100) ) - self.assertEqual("[1, 2...", utils.cut_list([1, 2, 3, 4, 5], 4)) - self.assertEqual("[1, 2...", utils.cut_list([1, 2, 3, 4, 5], 5)) - self.assertEqual("[1, 2, 3...", utils.cut_list([1, 2, 3, 4, 5], 6)) + self.assertEqual( + "['Hello, Mistral!', 'Hello, OpenStack!']", + utils.cut_list(list_, -1) + ) + + self.assertEqual("[1, 2...", utils.cut_list([1, 2, 3, 4, 5], 8)) + self.assertEqual("[1, 2,...", utils.cut_list([1, 2, 3, 4, 5], 9)) + self.assertEqual("[1, 2, 3...", utils.cut_list([1, 2, 3, 4, 5], 11)) self.assertRaises(ValueError, utils.cut_list, (1, 2)) + def test_cut_list_with_large_dict_of_str(self): + d = [str(i) for i in range(65535)] + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_list_with_large_dict_of_int(self): + d = [i for i in range(65535)] + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_list_with_large_dict_of_dict(self): + d = [{'value': str(i)} for i in range(65535)] + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_list_for_state_info(self): + d = [{'value': 'This is a string that exceeds 35 characters'} + for i in range(2000)] + s = utils.cut(d, 65500) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500))) + def test_cut_dict_with_strings(self): d = {'key1': 'value1', 'key2': 'value2'} - s = utils.cut_dict(d, 9) + s = utils.cut_dict(d, 13) self.assertIn(s, ["{'key1': '...", "{'key2': '..."]) - s = utils.cut_dict(d, 13) + s = utils.cut_dict(d, 15) self.assertIn(s, ["{'key1': 'va...", "{'key2': 'va..."]) - s = utils.cut_dict(d, 19) + s = utils.cut_dict(d, 22) self.assertIn( s, @@ -70,17 +198,55 @@ class TestUtils(tests_base.TestCase): ] ) + self.assertIn( + utils.cut_dict(d, -1), + [ + "{'key1': 'value1', 'key2': 'value2'}", + "{'key2': 'value2', 'key1': 'value1'}" + ] + ) + + self.assertRaises(ValueError, utils.cut_dict, (1, 2)) + def test_cut_dict_with_digits(self): d = {1: 2, 3: 4} - s = utils.cut_dict(d, 6) + s = utils.cut_dict(d, 10) self.assertIn(s, ["{1: 2, ...", "{3: 4, ..."]) - s = utils.cut_dict(d, 8) + s = utils.cut_dict(d, 11) self.assertIn(s, ["{1: 2, 3...", "{3: 4, 1..."]) s = utils.cut_dict(d, 100) self.assertIn(s, ["{1: 2, 3: 4}", "{3: 4, 1: 2}"]) + + def test_cut_dict_with_large_dict_of_str(self): + d = {} + for i in range(65535): + d[str(i)] = str(i) + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_dict_with_large_dict_of_int(self): + d = {} + for i in range(65535): + d[i] = i + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_dict_with_large_dict_of_dict(self): + d = {} + for i in range(65535): + d[i] = {'value': str(i)} + s = utils.cut(d, 65535) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535))) + + def test_cut_dict_for_state_info(self): + d = {} + for i in range(2000): + d[i] = {'value': 'This is a string that exceeds 35 characters'} + s = utils.cut(d, 65500) + self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500))) diff --git a/mistral_lib/utils/__init__.py b/mistral_lib/utils/__init__.py index 92dda4e..f08f15f 100644 --- a/mistral_lib/utils/__init__.py +++ b/mistral_lib/utils/__init__.py @@ -9,19 +9,197 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import functools +import json +import os +from os import path +import socket +import string +import sys +import threading + +import eventlet +from eventlet import corolocal +from oslo_log import log as logging +from oslo_utils import timeutils +from oslo_utils import uuidutils +import pkg_resources as pkg +import random + +# Thread local storage. +_th_loc_storage = threading.local() + +# TODO(rakhmerov): these two constants are misplaced. Utility methods +# should not be Mistral specific. They should be generic enough so to +# be moved to any other project w/o changes. +ACTION_TASK_TYPE = 'ACTION' +WORKFLOW_TASK_TYPE = 'WORKFLOW' + + +def generate_unicode_uuid(): + return uuidutils.generate_uuid() + + +def is_valid_uuid(uuid_string): + return uuidutils.is_uuid_like(uuid_string) + + +def _get_greenlet_local_storage(): + greenlet_id = corolocal.get_ident() + + greenlet_locals = getattr(_th_loc_storage, "greenlet_locals", None) + + if not greenlet_locals: + greenlet_locals = {} + _th_loc_storage.greenlet_locals = greenlet_locals + + if greenlet_id in greenlet_locals: + return greenlet_locals[greenlet_id] + else: + return None + + +def has_thread_local(var_name): + gl_storage = _get_greenlet_local_storage() + return gl_storage and var_name in gl_storage + + +def get_thread_local(var_name): + if not has_thread_local(var_name): + return None + + return _get_greenlet_local_storage()[var_name] + + +def set_thread_local(var_name, val): + if val is None and has_thread_local(var_name): + gl_storage = _get_greenlet_local_storage() + + # Delete variable from greenlet local storage. + if gl_storage: + del gl_storage[var_name] + + # Delete the entire greenlet local storage from thread local storage. + if gl_storage and len(gl_storage) == 0: + del _th_loc_storage.greenlet_locals[corolocal.get_ident()] + + if val is not None: + gl_storage = _get_greenlet_local_storage() + if not gl_storage: + gl_storage = _th_loc_storage.greenlet_locals[ + corolocal.get_ident()] = {} + + gl_storage[var_name] = val + + +def log_exec(logger, level=logging.DEBUG): + """Decorator for logging function execution. + + By default, target function execution is logged with DEBUG level. + """ + + def _decorator(func): + @functools.wraps(func) + def _logged(*args, **kw): + params_repr = ("[args=%s, kw=%s]" % (str(args), str(kw)) + if args or kw else "") + + func_repr = ("Called method [name=%s, doc='%s', params=%s]" % + (func.__name__, func.__doc__, params_repr)) + + logger.log(level, func_repr) + + return func(*args, **kw) + + _logged.__doc__ = func.__doc__ + + return _logged + + return _decorator + + +def merge_dicts(left, right, overwrite=True): + """Merges two dictionaries. + + Values of right dictionary recursively get merged into left dictionary. + :param left: Left dictionary. + :param right: Right dictionary. + :param overwrite: If False, left value will not be overwritten if exists. + """ + + if left is None: + return right + + if right is None: + return left + + for k, v in right.items(): + if k not in left: + left[k] = v + else: + left_v = left[k] + + if isinstance(left_v, dict) and isinstance(v, dict): + merge_dicts(left_v, v, overwrite=overwrite) + elif overwrite: + left[k] = v + + return left + + +def update_dict(left, right): + """Updates left dict with content from right dict + + :param left: Left dict. + :param right: Right dict. + :return: the updated left dictionary. + """ + + if left is None: + return right + + if right is None: + return left + + left.update(right) + + return left + + +def get_file_list(directory): + base_path = pkg.resource_filename("mistral", directory) + + return [path.join(base_path, f) for f in os.listdir(base_path) + if path.isfile(path.join(base_path, f))] def cut_dict(d, length=100): - """Truncates string representation of a dictionary for a given length. + """Removes dictionary entries according to the given length. - :param d: dictionary to truncate - :param length: amount of characters to truncate to - :return: string containing given length of characters from the dictionary + This method removes a number of entries, if needed, so that a + string representation would fit into the given length. + The intention of this method is to optimize truncation of string + representation for dictionaries where the exact precision is not + critically important. Otherwise, we'd always have to convert a dict + into a string first and then shrink it to a needed size which will + increase memory footprint and reduce performance in case of large + dictionaries (i.e. tens of thousands entries). + Note that the method, due to complexity of the algorithm, has some + non-zero precision which depends on exact keys and values placed into + the dict. So for some dicts their reduced string representations will + be only approximately equal to the given value (up to around several + chars difference). + + :param d: A dictionary. + :param length: A length limiting the dictionary string representation. + :return: A dictionary which is a subset of the given dictionary. """ if not isinstance(d, dict): raise ValueError("A dictionary is expected, got: %s" % type(d)) @@ -35,44 +213,42 @@ def cut_dict(d, length=100): v = str(value) # Processing key. - new_len = len(res) + len(k) + new_len = len(k) is_str = isinstance(key, str) if is_str: - new_len += 2 - - if new_len >= length: - res += "'%s..." % k[:length - new_len] if is_str else "%s..." % k + new_len += 2 # Account for the quotation marks + if 0 <= length <= new_len + len(res): + res += "'%s" % k if is_str else k break else: res += "'%s'" % k if is_str else k - res += ": " + + res += ": " # Processing value. - new_len = len(res) + len(v) + new_len = len(v) is_str = isinstance(value, str) if is_str: new_len += 2 - if new_len >= length: - res += "'%s..." % v[:length - new_len] if is_str else "%s..." % v - + if 0 <= length <= new_len + len(res): + res += "'%s" % v if is_str else v break else: res += "'%s'" % v if is_str else v - res += ', ' if idx < len(d) - 1 else '}' - if len(res) >= length: - res += '...' - - break + res += ', ' if idx < len(d) - 1 else '}' idx += 1 + if 0 <= length <= len(res) and res[length - 1] is not '}': + res = res[:length - 3] + '...' + return res @@ -98,13 +274,15 @@ def cut_list(l, length=100): if is_str: new_len += 2 - if new_len >= length: - res += "'%s..." % s[:length - new_len] if is_str else "%s..." % s - + if 0 <= length <= new_len: + res += "'%s" % s if is_str else s break else: res += "'%s'" % s if is_str else s - res += ', ' if idx < len(l) - 1 else ']' + res += ', ' if idx < len(l) - 1 else ']' + + if 0 <= length <= len(res) and res[length - 1] is not ']': + res = res[:length - 3] + '...' return res @@ -112,11 +290,11 @@ def cut_list(l, length=100): def cut_string(s, length=100): """Truncates a string for a given length. - :param s: string to truncate - :param length: amount of characters to truncate to - :return: string containing given length of characters - """ - if len(s) > length: + :param s: string to truncate + :param length: amount of characters to truncate to + :return: string containing given length of characters + """ + if 0 <= length < len(s): return "%s..." % s[:length] return s @@ -139,3 +317,170 @@ def cut(data, length=100): return cut_dict(data, length=length) return cut_string(str(data), length=length) + + +def cut_by_kb(data, kilobytes): + length = get_number_of_chars_from_kilobytes(kilobytes) + return cut(data, length) + + +def cut_by_char(data, length): + return cut(data, length) + + +def iter_subclasses(cls, _seen=None): + """Generator over all subclasses of a given class in depth first order.""" + + if not isinstance(cls, type): + raise TypeError('iter_subclasses must be called with new-style class' + ', not %.100r' % cls) + _seen = _seen or set() + + try: + subs = cls.__subclasses__() + except TypeError: # fails only when cls is type + subs = cls.__subclasses__(cls) + + for sub in subs: + if sub not in _seen: + _seen.add(sub) + yield sub + for _sub in iter_subclasses(sub, _seen): + yield _sub + + +def random_sleep(limit=1): + """Sleeps for a random period of time not exceeding the given limit. + + Mostly intended to be used by tests to emulate race conditions. + + :param limit: Float number of seconds that a sleep period must not exceed. + """ + + seconds = random.Random().randint(0, limit * 1000) * 0.001 + + print("Sleep: %s sec..." % seconds) + + eventlet.sleep(seconds) + + +class NotDefined(object): + """Marker of an empty value. + + In a number of cases None can't be used to express the semantics of + a not defined value because None is just a normal value rather than + a value set to denote that it's not defined. This class can be used + in such cases instead of None. + """ + + pass + + +def get_number_of_chars_from_kilobytes(kilobytes): + bytes_per_char = sys.getsizeof('s') - sys.getsizeof('') + total_number_of_chars = int(kilobytes * 1024 / bytes_per_char) + return total_number_of_chars + + +def get_dict_from_string(string, delimiter=','): + if not string: + return {} + + kv_dicts = [] + + for kv_pair_str in string.split(delimiter): + kv_str = kv_pair_str.strip() + kv_list = kv_str.split('=') + + if len(kv_list) > 1: + try: + value = json.loads(kv_list[1]) + except ValueError: + value = kv_list[1] + + kv_dicts += [{kv_list[0]: value}] + else: + kv_dicts += [kv_list[0]] + + return get_dict_from_entries(kv_dicts) + + +def get_dict_from_entries(entries): + """Transforms a list of entries into dictionary. + + :param entries: A list of entries. + If an entry is a dictionary the method simply updates the result + dictionary with its content. + If an entry is not a dict adds {entry, NotDefined} into the result. + """ + + result = {} + + for e in entries: + if isinstance(e, dict): + result.update(e) + else: + # NOTE(kong): we put NotDefined here as the value of + # param without value specified, to distinguish from + # the valid values such as None, ''(empty string), etc. + result[e] = NotDefined + + return result + + +def get_process_identifier(): + """Gets current running process identifier.""" + + return "%s_%s" % (socket.gethostname(), os.getpid()) + + +def utc_now_sec(): + """Returns current time and drops microseconds.""" + + return drop_microseconds(timeutils.utcnow()) + + +def drop_microseconds(date): + """Drops microseconds and returns date.""" + return date.replace(microsecond=0) + + +def datetime_to_str(val, sep=' '): + """Converts datetime value to string. + + If the given value is not an instance of datetime then the method + returns the same value. + + :param val: datetime value. + :param sep: Separator between date and time. + :return: Datetime as a string. + """ + if isinstance(val, datetime.datetime): + return val.isoformat(sep) + + return val + + +def datetime_to_str_in_dict(d, key, sep=' '): + """Converts datetime value in te given dict to string. + + :param d: A dictionary. + :param key: The key for which we need to convert the value. + :param sep: Separator between date and time. + """ + val = d.get(key) + + if val is not None: + d[key] = datetime_to_str(d[key], sep=sep) + + +def generate_string(length): + """Returns random string. + + :param length: the length of returned string + """ + + return ''.join(random.choice( + string.ascii_uppercase + string.digits) + for _ in range(length) + ) diff --git a/requirements.txt b/requirements.txt index b27c0da..c9e2689 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +eventlet!=0.20.1,>=0.20.0 # MIT +oslo.log>=3.36.0 # Apache-2.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 22c200a..15caea4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,6 +9,6 @@ sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD openstackdocstheme>=1.18.1 # Apache-2.0 oslotest>=3.2.0 # Apache-2.0 stestr>=2.0.0 # Apache-2.0 - +testtools>=2.2.0 # MIT # releasenotes reno>=2.5.0 # Apache-2.0