# Copyright (c) 2015 Hewlett-Packard. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # from __future__ import print_function from __future__ import division from __future__ import absolute_import import inspect import muranoclient.client from muranoclient.common import exceptions as murano_exceptions from oslo_log import log as logging from oslo_utils import uuidutils import six from congress.datasources import datasource_driver from congress.datasources import datasource_utils from congress.datasources import murano_classes logger = logging.getLogger(__name__) class MuranoDriver(datasource_driver.PollingDataSourceDriver, datasource_driver.ExecutionDriver): OBJECTS = "objects" PARENT_TYPES = "parent_types" PROPERTIES = "properties" RELATIONSHIPS = "relationships" CONNECTED = "connected" STATES = "states" ACTIONS = "actions" UNUSED_PKG_PROPERTIES = ['id', 'owner_id', 'description'] UNUSED_ENV_PROPERTIES = ['id', 'tenant_id'] APPS_TYPE_PREFIXES = ['io.murano.apps', 'io.murano.databases'] def __init__(self, name='', args=None): super(MuranoDriver, self).__init__(name, args=args) datasource_driver.ExecutionDriver.__init__(self) self.creds = args session = datasource_utils.get_keystone_session(self.creds) client_version = "1" self.murano_client = muranoclient.client.Client( client_version, session=session, endpoint_type='publicURL', service_type='application-catalog') self.add_executable_client_methods( self.murano_client, 'muranoclient.v1.') logger.debug("Successfully created murano_client") self.action_call_returns = [] self._init_end_start_poll() @staticmethod def get_datasource_info(): result = {} result['id'] = 'murano' result['description'] = ('Datasource driver that interfaces with ' 'murano') result['config'] = datasource_utils.get_openstack_required_config() result['secret'] = ['password'] return result def update_from_datasource(self): """Called when it is time to pull new data from this datasource. Sets self.state[tablename] = for every tablename exported by this datasource. """ self.state[self.STATES] = set() self.state[self.OBJECTS] = set() self.state[self.PROPERTIES] = set() self.state[self.PARENT_TYPES] = set() self.state[self.RELATIONSHIPS] = set() self.state[self.CONNECTED] = set() self.state[self.ACTIONS] = dict() # Workaround for 401 error issue try: # Moves _translate_packages above translate_services to # make use of properties table in translate_services logger.debug("Murano grabbing packages") packages = self.murano_client.packages.list() self._translate_packages(packages) logger.debug("Murano grabbing environments") environments = self.murano_client.environments.list() self._translate_environments(environments) self._translate_services(environments) self._translate_deployments(environments) self._translate_connected() except murano_exceptions.HTTPException: raise @classmethod def get_schema(cls): """Returns a dictionary of table schema. The dictionary mapping tablenames to the list of column names for that table. Both tablenames and columnnames are strings. """ d = {} d[cls.OBJECTS] = ('object_id', 'owner_id', 'type') # parent_types include not only the type of object's immediate # parent but also all of its ancestors and its own type. The # additional info helps writing better datalog rules. d[cls.PARENT_TYPES] = ('id', 'parent_type') d[cls.PROPERTIES] = ('owner_id', 'name', 'value') d[cls.RELATIONSHIPS] = ('source_id', 'target_id', 'name') d[cls.CONNECTED] = ('source_id', 'target_id') d[cls.STATES] = ('id', 'state') return d def _translate_environments(self, environments): """Translate the environments into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from environments """ logger.debug("_translate_environments: %s", environments) if not environments: return self.state[self.STATES] = set() if self.OBJECTS not in self.state: self.state[self.OBJECTS] = set() if self.PROPERTIES not in self.state: self.state[self.PROPERTIES] = set() if self.PARENT_TYPES not in self.state: self.state[self.PARENT_TYPES] = set() if self.RELATIONSHIPS not in self.state: self.state[self.RELATIONSHIPS] = set() if self.CONNECTED not in self.state: self.state[self.CONNECTED] = set() env_type = 'io.murano.Environment' for env in environments: self.state[self.OBJECTS].add( (env.id, env.tenant_id, env_type)) self.state[self.STATES].add((env.id, env.status)) parent_types = self._get_parent_types(env_type) self._add_parent_types(env.id, parent_types) for key, value in env.to_dict().items(): if key in self.UNUSED_ENV_PROPERTIES: continue self._add_properties(env.id, key, value) def _translate_services(self, environments): """Translate the environment services into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from services """ logger.debug("Murano grabbing environments services") if not environments: return for env in environments: services = self.murano_client.services.list(env.id) self._translate_environment_services(services, env.id) def _translate_environment_services(self, services, env_id): """Translate the environment services into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from services """ # clean actions for given environment if self.ACTIONS not in self.state: self.state[self.ACTIONS] = dict() env_actions = self.state[self.ACTIONS][env_id] = set() if not services: return for s in services: s_dict = s.to_dict() s_id = s_dict['?']['id'] s_type = s_dict['?']['type'] self.state[self.OBJECTS].add((s_id, env_id, s_type)) for key, value in s_dict.items(): if key in ['instance', '?']: continue self._add_properties(s_id, key, value) self._add_relationships(s_id, key, value) parent_types = self._get_parent_types(s_type) self._add_parent_types(s_id, parent_types) self._add_relationships(env_id, 'services', s_id) self._translate_service_action(s_dict, env_actions) if 'instance' not in s_dict: continue # populate service instance si_dict = s.instance si_id = si_dict['?']['id'] si_type = si_dict['?']['type'] self.state[self.OBJECTS].add((si_id, s_id, si_type)) for key, value in si_dict.items(): if key in ['?']: continue self._add_properties(si_id, key, value) if key not in ['image']: # there's no murano image object in the environment, # therefore glance 'image' relationship is irrelevant # at this point. self._add_relationships(si_id, key, value) # There's a relationship between the service and instance self._add_relationships(s_id, 'instance', si_id) parent_types = self._get_parent_types(si_type) self._add_parent_types(si_id, parent_types) self._translate_service_action(si_dict, env_actions) def _translate_service_action(self, obj_dict, env_actions): """Translates environment's object actions to env_actions structure. env_actions: [(obj_id, action_id, action_name, enabled)] :param: obj_dict: object dictionary :param: env_actions: set of environment actions """ obj_id = obj_dict['?']['id'] if '_actions' in obj_dict['?']: o_actions = obj_dict['?']['_actions'] if not o_actions: return for action_id, action_value in o_actions.items(): action_name = action_value.get('name', '') enabled = action_value.get('enabled', False) action = (obj_id, action_id, action_name, enabled) env_actions.add(action) # TODO(tranldt): support action arguments. # If action arguments are included in '_actions', # they can be populated into tables. def _translate_deployments(self, environments): """Translate the environment deployments into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from deployments """ if not environments: return for env in environments: deployments = self.murano_client.deployments.list(env.id) self._translate_environment_deployments(deployments, env.id) def _translate_environment_deployments(self, deployments, env_id): """Translate the environment deployments into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from deployments """ if not deployments: return for d in deployments: if 'defaultNetworks' not in d.description: continue default_networks = d.description['defaultNetworks'] net_id = None if 'environment' in default_networks: net_id = default_networks['environment']['?']['id'] net_type = default_networks['environment']['?']['type'] self.state[self.OBJECTS].add((net_id, env_id, net_type)) parent_types = self._get_parent_types(net_type) self._add_parent_types(net_id, parent_types) for key, value in default_networks['environment'].items(): if key in ['?']: continue self._add_properties(net_id, key, value) if not net_id: continue self._add_relationships(env_id, 'defaultNetworks', net_id) for key, value in default_networks.items(): if key in ['environment']: # data from environment already populated continue new_key = 'defaultNetworks.' + key self._add_properties(net_id, new_key, value) # services from deployment are not of interest because the same # info is obtained from services API def _translate_packages(self, packages): """Translate the packages into tables. Assigns self.state[tablename] for all those TABLENAMEs generated from packages/applications """ # packages is a generator type if not packages: return if self.OBJECTS not in self.state: self.state[self.OBJECTS] = set() if self.PROPERTIES not in self.state: self.state[self.PROPERTIES] = set() for pkg in packages: logger.debug("pkg=%s", pkg.to_dict()) pkg_type = pkg.type if pkg.type == 'Application': pkg_type = 'io.murano.Application' self.state[self.OBJECTS].add((pkg.id, pkg.owner_id, pkg_type)) for key, value in pkg.to_dict().items(): if key in self.UNUSED_PKG_PROPERTIES: continue self._add_properties(pkg.id, key, value) def _add_properties(self, obj_id, key, value): """Add a set of (obj_id, key, value) to properties table. :param: obj_id: uuid of object :param: key: property name. For the case value is a list, the same key is used for multiple values. :param: value: property value. If value is a dict, the nested properties will be mapped using dot notation. """ if value is None or value == '': return if isinstance(value, dict): for k, v in value.items(): new_key = key + "." + k self._add_properties(obj_id, new_key, v) elif isinstance(value, list): if len(value) == 0: return for item in value: self.state[self.PROPERTIES].add( (obj_id, key, item)) else: self.state[self.PROPERTIES].add( (obj_id, key, value)) def _add_relationships(self, obj_id, key, value): """Add a set of (obj_id, value, key) to relationships table. :param: obj_id: source uuid :param: key: relationship name :param: value: target uuid """ if (not isinstance(value, six.string_types) or not uuidutils.is_uuid_like(value)): return logger.debug("Relationship: source = %s, target = %s, rel_name = %s" % (obj_id, value, key)) self.state[self.RELATIONSHIPS].add((obj_id, value, key)) def _transitive_closure(self): """Computes transitive closure on a directed graph. In other words computes reachability within the graph. E.g. {(1, 2), (2, 3)} -> {(1, 2), (2, 3), (1, 3)} (1, 3) was added because there is path from 1 to 3 in the graph. """ closure = self.state[self.CONNECTED] while True: # Attempts to discover new transitive relations # by joining 2 subsequent relations/edges within the graph. new_relations = {(x, w) for x, y in closure for q, w in closure if q == y} # Creates union with already discovered relations. closure_until_now = closure | new_relations # If no new relations were discovered in last cycle # the computation is finished. if closure_until_now == closure: self.state[self.CONNECTED] = closure break closure = closure_until_now def _add_connected(self, source_id, target_id): """Looks up the target_id in objects and add links to connected table. Adds sets of (source_id, target_id) to connected table along with its indirections. :param: source_id: source uuid :param: target_id: target uuid """ for row in self.state[self.OBJECTS]: if row[1] == target_id: self.state[self.CONNECTED].add((row[1], row[0])) self.state[self.CONNECTED].add((source_id, row[0])) self.state[self.CONNECTED].add((source_id, target_id)) def _translate_connected(self): """Translates relationships table into connected table.""" for row in self.state[self.RELATIONSHIPS]: self._add_connected(row[0], row[1]) self._transitive_closure() def _add_parent_types(self, obj_id, parent_types): """Add sets of (obj_id, parent_type) to parent_types table. :param: obj_id: uuid of object :param: parent_types: list of parent type string """ if parent_types: for p_type in parent_types: self.state[self.PARENT_TYPES].add((obj_id, p_type)) def _get_package_type(self, class_name): """Determine whether obj_type is an Application or Library. :param: class_name: service/application class name e.g. io.murano.apps.linux.Telnet. :return: - package type (e.g. 'Application') if found. - None if no package type found. """ pkg_type = None if self.PROPERTIES in self.state: idx_uuid = 0 idx_value = 2 uuid = None for row in self.state[self.PROPERTIES]: if 'class_definitions' in row and class_name in row: uuid = row[idx_uuid] break if uuid: for row in self.state[self.PROPERTIES]: if 'type' in row and uuid == row[idx_uuid]: pkg_type = row[idx_value] # If the package is removed after deployed, its properties # are not known and so above search will fail. In that case # let's check for class_name prefix as the last resort. if not pkg_type: for prefix in self.APPS_TYPE_PREFIXES: if prefix in class_name: pkg_type = 'Application' break return pkg_type def _get_parent_types(self, obj_type): """Get class types of all OBJ_TYPE's parents including itself. Look up the hierarchy of OBJ_TYPE and return types of all its ancestor including its own type. :param: obj_type: """ class_types = [] p = lambda x: inspect.isclass(x) g = inspect.getmembers(murano_classes, p) for name, cls in g: logger.debug("%s: %s" % (name, cls)) if (cls is murano_classes.IOMuranoApps and self._get_package_type(obj_type) == 'Application'): cls.name = obj_type if 'get_parent_types' in dir(cls): class_types = cls.get_parent_types(obj_type) if class_types: break return class_types def _call_murano_action(self, environment_id, object_id, action_name): """Invokes action of object in Murano environment. :param: environment_id: uuid :param: object_id: uuid :param: action_name: string """ # get action id using object_id, env_id and action name logger.debug("Requested Murano action invoke %s on %s in %s", action_name, object_id, environment_id) if (not self.state[self.ACTIONS] or environment_id not in self.state[self.ACTIONS]): logger.warning('Datasource "%s" found no actions for ' 'environment "%s"', self.name, environment_id) return env_actions = self.state[self.ACTIONS][environment_id] for env_action in env_actions: ea_obj_id, ea_action_id, ea_action_name, ea_enabled = env_action if (object_id == ea_obj_id and action_name == ea_action_name and ea_enabled): logger.debug("Invoking Murano action_id = %s, action_name %s", ea_action_id, ea_action_name) # TODO(tranldt): support action arguments task_id = self.murano_client.actions.call(environment_id, ea_action_id) logger.debug("Murano action invoked %s - task id %s", ea_action_id, task_id) self.action_call_returns.append(task_id) def execute(self, action, action_args): """Overwrite ExecutionDriver.execute().""" logger.info("%s:: executing %s on %s", self.name, action, action_args) self.action_call_returns = [] positional_args = action_args.get('positional', []) logger.debug('Processing action execution: action = %s, ' 'positional args = %s', action, positional_args) try: env_id = positional_args[0] obj_id = positional_args[1] action_name = positional_args[2] self._call_murano_action(env_id, obj_id, action_name) except Exception as e: logger.exception(str(e))