deb-murano/murano/dsl/helpers.py

#    Copyright (c) 2014 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import contextlib
import functools
import gc
import inspect
import itertools
import re
import sys
import uuid
import weakref


import eventlet.greenpool
import eventlet.greenthread
import semantic_version
import six
from yaql.language import contexts
import yaql.language.exceptions
import yaql.language.expressions
from yaql.language import utils as yaqlutils


from murano.dsl import constants
from murano.dsl import dsl_types
from murano.dsl import exceptions

_threads_sequencer = 0
# type string: ns.something.MyApp[/1.2.3-alpha][@my.package.fqn]
TYPE_RE = re.compile(r'([a-zA-Z0-9_.]+)(?:/([^@]+))?(?:@([a-zA-Z0-9_.]+))?$')


def evaluate(value, context, freeze=True):
    list_type = tuple if freeze else list
    dict_type = yaqlutils.FrozenDict if freeze else dict
    set_type = frozenset if freeze else set

    if isinstance(value, (dsl_types.YaqlExpression,
                          yaql.language.expressions.Statement)):
        return value(context)
    elif isinstance(value, yaqlutils.MappingType):
        return dict_type(
            (evaluate(d_key, context, freeze),
             evaluate(d_value, context, freeze))
            for d_key, d_value in six.iteritems(value))
    elif yaqlutils.is_sequence(value):
        return list_type(evaluate(t, context, freeze) for t in value)
    elif isinstance(value, yaqlutils.SetType):
        return set_type(evaluate(t, context, freeze) for t in value)
    elif yaqlutils.is_iterable(value):
        return list_type(
            evaluate(t, context, freeze)
            for t in yaqlutils.limit_iterable(
                value, constants.ITERATORS_LIMIT))
    elif isinstance(value, dsl_types.MuranoObjectInterface):
        return value.object
    else:
        return value


def merge_lists(list1, list2):
    result = []
    for item in list1 + list2:
        if item not in result:
            result.append(item)
    return result


def merge_dicts(dict1, dict2, max_levels=0):
    result = {}
    for key, value1 in dict1.items():
        result[key] = value1
        if key in dict2:
            value2 = dict2[key]
            if type(value2) != type(value1):
                if ((isinstance(value1,
                                six.string_types) or value1 is None) and
                        (isinstance(value2,
                                    six.string_types) or value2 is None)):
                    continue
                raise TypeError()
            if max_levels != 1 and isinstance(value2, dict):
                result[key] = merge_dicts(
                    value1, value2,
                    0 if max_levels == 0 else max_levels - 1)
            elif max_levels != 1 and isinstance(value2, list):
                result[key] = merge_lists(value1, value2)
            else:
                result[key] = value2
    for key, value1 in dict2.items():
        if key not in result:
            result[key] = value1
    return result


def generate_id():
    return uuid.uuid4().hex


def parallel_select(collection, func, limit=1000):
    # workaround for eventlet issue 232
    # https://github.com/eventlet/eventlet/issues/232
    context = get_context()
    object_store = get_object_store()

    def wrapper(element):
        try:
            with with_object_store(object_store), contextual(context):
                return func(element), False, None
        except Exception as e:
            return e, True, sys.exc_info()[2]

    gpool = eventlet.greenpool.GreenPool(limit)
    result = list(gpool.imap(wrapper, collection))
    try:
        exception = next(t for t in result if t[1])
    except StopIteration:
        return map(lambda t: t[0], result)
    else:
        six.reraise(type(exception[0]), exception[0], exception[2])


def enum(**enums):
    return type('Enum', (), enums)


def get_context():
    current_thread = eventlet.greenthread.getcurrent()
    return getattr(current_thread, constants.TL_CONTEXT, None)


def get_executor():
    store = get_object_store()
    return None if store is None else store.executor


def get_type(context=None):
    context = context or get_context()
    return context[constants.CTX_TYPE]


def get_execution_session():
    executor = get_executor()
    return None if executor is None else executor.execution_session


def get_object_store():
    current_thread = eventlet.greenthread.getcurrent()
    return getattr(current_thread, constants.TL_OBJECT_STORE, None)


def get_package_loader():
    executor = get_executor()
    return None if executor is None else executor.package_loader


def get_this(context=None):
    context = context or get_context()
    return context[constants.CTX_THIS]


def get_caller_context(context=None):
    context = context or get_context()
    return context[constants.CTX_CALLER_CONTEXT]


def get_attribute_store():
    executor = get_executor()
    return None if executor is None else executor.attribute_store


def get_current_instruction(context=None):
    context = context or get_context()
    return context[constants.CTX_CURRENT_INSTRUCTION]


def get_current_method(context=None):
    context = context or get_context()
    return context[constants.CTX_CURRENT_METHOD]


def get_yaql_engine(context=None):
    context = context or get_context()
    return None if context is None else context[constants.CTX_YAQL_ENGINE]


def get_current_exception(context=None):
    context = context or get_context()
    return context[constants.CTX_CURRENT_EXCEPTION]


def are_property_modifications_allowed(context=None):
    context = context or get_context()
    return context[constants.CTX_ALLOW_PROPERTY_WRITES] or False


def get_names_scope(context=None):
    context = context or get_context()
    return context[constants.CTX_NAMES_SCOPE]


def get_class(name, context=None):
    context = context or get_context()
    murano_type = get_names_scope(context)
    name = murano_type.namespace_resolver.resolve_name(name)
    return murano_type.package.find_class(name)


def get_contract_passkey():
    current_thread = eventlet.greenthread.getcurrent()
    return getattr(current_thread, constants.TL_CONTRACT_PASSKEY, None)


def is_objects_dry_run_mode():
    current_thread = eventlet.greenthread.getcurrent()
    return bool(getattr(current_thread, constants.TL_OBJECTS_DRY_RUN, False))


def get_current_thread_id():
    global _threads_sequencer

    current_thread = eventlet.greenthread.getcurrent()
    thread_id = getattr(current_thread, constants.TL_ID, None)
    if thread_id is None:
        thread_id = 'T' + str(_threads_sequencer)
        _threads_sequencer += 1
        setattr(current_thread, constants.TL_ID, thread_id)
    return thread_id


@contextlib.contextmanager
def thread_local_attribute(name, value):
    current_thread = eventlet.greenthread.getcurrent()
    old_value = getattr(current_thread, name, None)
    if value is not None:
        setattr(current_thread, name, value)
    elif hasattr(current_thread, name):
        delattr(current_thread, name)
    try:
        yield
    finally:
        if old_value is not None:
            setattr(current_thread, name, old_value)
        elif hasattr(current_thread, name):
            delattr(current_thread, name)


def contextual(ctx):
    return thread_local_attribute(constants.TL_CONTEXT, ctx)


def with_object_store(object_store):
    return thread_local_attribute(constants.TL_OBJECT_STORE, object_store)


def parse_version_spec(version_spec):
    if isinstance(version_spec, semantic_version.Spec):
        return normalize_version_spec(version_spec)
    if isinstance(version_spec, semantic_version.Version):
        return normalize_version_spec(
            semantic_version.Spec('==' + str(version_spec)))
    if not version_spec:
        version_spec = '0'
    version_spec = re.sub('\s+', '', str(version_spec))

    # NOTE(kzaitsev): semantic_version 2.3.X thinks that '=0' is not
    # a valid version spec and only accepts '==0', this regexp adds
    # an extra '=' before such specs
    version_spec = re.sub(r'^=(\d)', r'==\1', version_spec)

    if version_spec[0].isdigit():
        version_spec = '==' + str(version_spec)
    version_spec = semantic_version.Spec(version_spec)
    return normalize_version_spec(version_spec)


def parse_version(version):
    if isinstance(version, semantic_version.Version):
        return version
    if not version:
        version = '0'
    return semantic_version.Version.coerce(str(version))


def traverse(seed, producer=None, track_visited=True):
    if not yaqlutils.is_iterable(seed):
        seed = [seed]
    visited = None if not track_visited else set()
    queue = collections.deque(seed)
    while queue:
        item = queue.popleft()
        if track_visited:
            if item in visited:
                continue
            visited.add(item)
        produced = (yield item)
        if produced is None and producer:
            produced = producer(item)
        if produced:
            queue.extend(produced)


def cast(obj, murano_class, pov_or_version_spec=None):
    if isinstance(obj, dsl_types.MuranoObjectInterface):
        obj = obj.object
    if isinstance(pov_or_version_spec, dsl_types.MuranoType):
        pov_or_version_spec = pov_or_version_spec.package
    elif isinstance(pov_or_version_spec, six.string_types):
        pov_or_version_spec = parse_version_spec(pov_or_version_spec)

    if isinstance(murano_class, dsl_types.MuranoTypeReference):
        murano_class = murano_class.type
    if isinstance(murano_class, dsl_types.MuranoType):
        if pov_or_version_spec is None:
            pov_or_version_spec = parse_version_spec(murano_class.version)
        murano_class = murano_class.name

    candidates = []
    for cls in itertools.chain((obj.type,), obj.type.ancestors()):
        if cls.name != murano_class:
            continue
        elif isinstance(pov_or_version_spec, semantic_version.Version):
            if cls.version != pov_or_version_spec:
                continue
        elif isinstance(pov_or_version_spec, semantic_version.Spec):
            if cls.version not in pov_or_version_spec:
                continue
        elif isinstance(pov_or_version_spec, dsl_types.MuranoPackage):
            requirement = pov_or_version_spec.requirements.get(
                cls.package.name)
            if requirement is None:
                raise exceptions.NoClassFound(murano_class)
            if cls.version not in requirement:
                continue
        elif pov_or_version_spec is not None:
            raise ValueError('pov_or_version_spec of unsupported '
                             'type {0}'.format(type(pov_or_version_spec)))
        candidates.append(cls)
    if not candidates:
        raise exceptions.NoClassFound(murano_class)
    elif len(candidates) > 1:
        raise exceptions.AmbiguousClassName(murano_class)
    return obj.cast(candidates[0])


def is_instance_of(obj, class_name, pov_or_version_spec=None):
    if not isinstance(obj, (dsl_types.MuranoObject,
                            dsl_types.MuranoObjectInterface)):
        return False
    try:
        cast(obj, class_name, pov_or_version_spec)
        return True
    except (exceptions.NoClassFound, exceptions.AmbiguousClassName):
        return False


def memoize(func):
    cache = {}
    return get_memoize_func(func, cache)


def get_memoize_func(func, cache):
    @functools.wraps(func)
    def wrap(*args):
        if args not in cache:
            result = func(*args)
            cache[args] = result
            return result
        else:
            return cache[args]
    return wrap


def normalize_version_spec(version_spec):
    def coerce(v):
        return semantic_version.Version('{0}.{1}.{2}'.format(
            v.major, v.minor or 0, v.patch or 0
        ))

    def increment(v):
        # NOTE(ativelkov): replace these implementations with next_minor() and
        # next_major() calls when the semantic_version is updated in global
        # requirements.
        if v.minor is None:
            return semantic_version.Version(
                '.'.join(str(x) for x in [v.major + 1, 0, 0]))
        else:
            return semantic_version.Version(
                '.'.join(str(x) for x in [v.major, v.minor + 1, 0]))

    def extend(v):
        return semantic_version.Version(str(v) + '-0')

    transformations = {
        '>': [('>=', (increment, extend))],
        '>=': [('>=', (coerce,))],
        '<': [('<', (coerce, extend))],
        '<=': [('<', (increment, extend))],
        '!=': [('>=', (increment, extend))],
        '==': [('>=', (coerce,)), ('<', (increment, coerce, extend))]
    }

    new_parts = []
    for item in version_spec.specs:
        if item.kind == '*':
            continue
        elif item.spec.patch is not None:
            new_parts.append(str(item))
        else:
            for op, funcs in transformations[item.kind]:
                new_parts.append('{0}{1}'.format(
                    op,
                    six.moves.reduce(lambda v, f: f(v), funcs, item.spec)
                ))
    if not new_parts:
        return semantic_version.Spec('*')
    return semantic_version.Spec(*new_parts)


semver_to_api_map = {
    '>': 'gt',
    '>=': 'ge',
    '<': 'lt',
    '<=': 'le',
    '!=': 'ne',
    '==': 'eq'
}


def breakdown_spec_to_query(normalized_spec):
    res = []
    for item in normalized_spec.specs:
        if item.kind == '*':
            continue
        else:
            res.append("%s:%s" % (semver_to_api_map[item.kind],
                                  item.spec))
    return res


def link_contexts(parent_context, context):
    if not context:
        return parent_context
    return contexts.LinkedContext(parent_context, context)


def inspect_is_static(cls, name):
    m = cls.__dict__.get(name)
    if m is None:
        return False
    return isinstance(m, staticmethod)


def inspect_is_classmethod(cls, name):
    m = cls.__dict__.get(name)
    if m is None:
        return False
    return isinstance(m, classmethod)


def inspect_is_method(cls, name):
    m = getattr(cls, name, None)
    if m is None:
        return False
    return ((inspect.isfunction(m) or inspect.ismethod(m)) and not
            inspect_is_static(cls, name) and not
            inspect_is_classmethod(cls, name))


def inspect_is_property(cls, name):
    m = getattr(cls, name, None)
    if m is None:
        return False
    return inspect.isdatadescriptor(m)


def updated_dict(d, val):
    if d is None:
        d = {}
    else:
        d = d.copy()
    if val is not None:
        d.update(val)
    return d


def resolve_type(value, scope_type, return_reference=False):
    if value is None:
        return None
    if isinstance(scope_type, dsl_types.MuranoTypeReference):
        scope_type = scope_type.type
    if not isinstance(value, (dsl_types.MuranoType,
                              dsl_types.MuranoTypeReference)):
        name = scope_type.namespace_resolver.resolve_name(value)
        result = scope_type.package.find_class(name)
    else:
        result = value

    if isinstance(result, dsl_types.MuranoTypeReference):
        if return_reference:
            return result
        return result.type
    elif return_reference:
        return result.get_reference()
    return result


def parse_object_definition(spec, scope_type, context):
    if not isinstance(spec, yaqlutils.MappingType):
        return None

    if context:
        spec = evaluate(spec, context, freeze=False)
    else:
        spec = spec.copy()
    system_data = None
    type_obj = None
    props = {}
    ns_resolver = scope_type.namespace_resolver if scope_type else None
    for key in spec:
        if (ns_resolver and ns_resolver.is_typename(key, False) or
                isinstance(key, (dsl_types.MuranoTypeReference,
                                 dsl_types.MuranoType))):
            type_obj = resolve_type(key, scope_type)
            props = spec.pop(key) or {}
            system_data = spec
            break
    if system_data is None:
        props = spec
        if '?' in spec:
            system_data = spec.pop('?')
            obj_type = system_data.get('type')
            if isinstance(obj_type, dsl_types.MuranoTypeReference):
                type_obj = obj_type.type
            elif isinstance(obj_type, dsl_types.MuranoType):
                type_obj = obj_type
            elif obj_type:
                type_str, version_str, package_str = parse_type_string(
                    obj_type,
                    system_data.get('classVersion'),
                    system_data.get('package')
                )
                version_spec = parse_version_spec(version_str)
                package_loader = get_package_loader()
                if package_str:
                    package = package_loader.load_package(
                        package_str, version_spec)
                else:
                    package = package_loader.load_class_package(
                        type_str, version_spec)
                type_obj = package.find_class(type_str, False)
        else:
            system_data = {}

    return {
        'type': type_obj,
        'properties': yaqlutils.filter_parameters_dict(props),
        'id': system_data.get('id'),
        'name': system_data.get('name'),
        'metadata': system_data.get('metadata'),
        'destroyed': system_data.get('destroyed', False),
        'dependencies': system_data.get('dependencies', {}),
        'extra': {
            key: value for key, value in six.iteritems(system_data)
            if key.startswith('_')
        }
    }


def assemble_object_definition(parsed, model_format=dsl_types.DumpTypes.Mixed):
    if model_format == dsl_types.DumpTypes.Inline:
        result = {
            parsed['type']: parsed['properties'],
            'id': parsed['id'],
            'name': parsed['name'],
            'metadata': parsed['metadata'],
            'dependencies': parsed['dependencies'],
            'destroyed': parsed['destroyed']
        }
        result.update(parsed['extra'])
        return result
    result = parsed['properties']
    header = {
        'id': parsed['id'],
        'name': parsed['name'],
        'metadata': parsed['metadata']
    }
    if parsed['destroyed']:
        header['destroyed'] = True
    header.update(parsed['extra'])
    result['?'] = header
    if model_format == dsl_types.DumpTypes.Mixed:
        header['type'] = parsed['type']
        return result
    elif model_format == dsl_types.DumpTypes.Serializable:
        cls = parsed['type']
        if cls:
            header['type'] = format_type_string(cls)
        return result
    else:
        raise ValueError('Invalid Serialization Type')


def function(c):
    if hasattr(c, 'im_func'):
        return c.im_func
    return c


def list_value(v):
    if v is None:
        return []
    if not yaqlutils.is_sequence(v):
        v = [v]
    return v


def weak_proxy(obj):
    if obj is None or isinstance(obj, weakref.ProxyType):
        return obj
    if isinstance(obj, weakref.ReferenceType):
        obj = obj()
    return weakref.proxy(obj)


def weak_ref(obj):
    class MuranoObjectWeakRef(weakref.ReferenceType):
        def __init__(self, murano_object):
            self.ref = weakref.ref(murano_object)
            self.object_id = murano_object.object_id

        def __call__(self):
            res = self.ref()
            if not res:
                object_store = get_object_store()
                if object_store:
                    res = object_store.get(self.object_id)
                    if res:
                        self.ref = weakref.ref(res)
            return res

    if obj is None or isinstance(obj, weakref.ReferenceType):
        return obj

    if isinstance(obj, dsl_types.MuranoObject):
        return MuranoObjectWeakRef(obj)
    return weakref.ref(obj)


def parse_type_string(type_str, default_version, default_package):
    res = TYPE_RE.match(type_str)
    if res is None:
        return None
    parsed_type = res.group(1)
    parsed_version = res.group(2)
    parsed_package = res.group(3)
    return (
        parsed_type,
        default_version if parsed_version is None else parsed_version,
        default_package if parsed_package is None else parsed_package
    )


def format_type_string(type_obj):
    if isinstance(type_obj, dsl_types.MuranoTypeReference):
        type_obj = type_obj.type
    if isinstance(type_obj, dsl_types.MuranoType):
        return '{0}/{1}@{2}'.format(
            type_obj.name, type_obj.version, type_obj.package.name)
    else:
        raise ValueError('Invalid argument')


def patch_dict(dct, path, value):
    parts = path.split('.')
    for i in range(len(parts) - 1):
        if not isinstance(dct, dict):
            dct = None
            break
        dct = dct.get(parts[i])
    if isinstance(dct, dict):
        if value is yaqlutils.NO_VALUE:
            dct.pop(parts[-1])
        else:
            dct[parts[-1]] = value


def format_scalar(value):
    if isinstance(value, six.string_types):
        return "'{0}'".format(value)
    return six.text_type(value)


def is_passkey(value):
    passkey = get_contract_passkey()
    return passkey is not None and value is passkey


def find_object_owner(obj, predicate):
    p = obj.owner
    while p:
        if predicate(p):
            return p
        p = p.owner
    return None


# This function is not intended to be used in the code but is very useful
# for debugging object reference leaks
def walk_gc(obj, towards, handler):
    visited = set()
    queue = collections.deque([(obj, [])])
    while queue:
        item, trace = queue.popleft()
        if id(item) in visited:
            continue
        if handler(item):
            if towards:
                yield trace + [item]
            else:
                yield [item] + trace

        visited.add(id(item))
        if towards:
            queue.extend(
                [(t, trace + [item]) for t in gc.get_referrers(item)]
            )
        else:
            queue.extend(
                [(t, [item] + trace) for t in gc.get_referents(item)]
            )