c8b3774a16
Move the conversion to agnostic-compatible types from datasource_driver to agnostic. Internal change only; external API unaffected. The datalog compiler and the agnostic policy engine in Congress do not support booleans and Nones/nulls. To work with that design, the data source drivers currently convert booleans and Nones into strings. This works well for agnostic, but unnecessarily limits the types handled by the data source drivers. In the spirit of the original architecture to support policy engines in addition to agnostic, this change expands the types handled by the data source drivers and the Congress data bus to all the JSON scalar types, de-coupling from the more limited set of types accepted by agnostic. This way, other policy engines can independently choose which subset of the data types to work with. This would be a small first step toward integrating other policy engines such as those based on SQL, z3, or Open Policy Agent. Change-Id: Ia63614a84742833c75ca3cadec7db48a44200e9d
510 lines
20 KiB
Python
510 lines
20 KiB
Python
# Copyright (c) 2015 Hewlett-Packard. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import absolute_import
|
|
|
|
import inspect
|
|
|
|
import muranoclient.client
|
|
from muranoclient.common import exceptions as murano_exceptions
|
|
from oslo_log import log as logging
|
|
from oslo_utils import uuidutils
|
|
import six
|
|
|
|
from congress.datasources import datasource_driver
|
|
from congress.datasources import datasource_utils
|
|
from congress.datasources import murano_classes
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MuranoDriver(datasource_driver.PollingDataSourceDriver,
|
|
datasource_driver.ExecutionDriver):
|
|
OBJECTS = "objects"
|
|
PARENT_TYPES = "parent_types"
|
|
PROPERTIES = "properties"
|
|
RELATIONSHIPS = "relationships"
|
|
CONNECTED = "connected"
|
|
STATES = "states"
|
|
ACTIONS = "actions"
|
|
UNUSED_PKG_PROPERTIES = ['id', 'owner_id', 'description']
|
|
UNUSED_ENV_PROPERTIES = ['id', 'tenant_id']
|
|
APPS_TYPE_PREFIXES = ['io.murano.apps', 'io.murano.databases']
|
|
|
|
def __init__(self, name='', args=None):
|
|
super(MuranoDriver, self).__init__(name, args=args)
|
|
datasource_driver.ExecutionDriver.__init__(self)
|
|
self.creds = args
|
|
session = datasource_utils.get_keystone_session(self.creds)
|
|
client_version = "1"
|
|
self.murano_client = muranoclient.client.Client(
|
|
client_version, session=session, endpoint_type='publicURL',
|
|
service_type='application-catalog')
|
|
self.add_executable_client_methods(
|
|
self.murano_client,
|
|
'muranoclient.v1.')
|
|
logger.debug("Successfully created murano_client")
|
|
|
|
self.action_call_returns = []
|
|
self._init_end_start_poll()
|
|
|
|
@staticmethod
|
|
def get_datasource_info():
|
|
result = {}
|
|
result['id'] = 'murano'
|
|
result['description'] = ('Datasource driver that interfaces with '
|
|
'murano')
|
|
result['config'] = datasource_utils.get_openstack_required_config()
|
|
result['secret'] = ['password']
|
|
return result
|
|
|
|
def update_from_datasource(self):
|
|
"""Called when it is time to pull new data from this datasource.
|
|
|
|
Sets self.state[tablename] = <set of tuples of strings/numbers>
|
|
for every tablename exported by this datasource.
|
|
"""
|
|
self.state[self.STATES] = set()
|
|
self.state[self.OBJECTS] = set()
|
|
self.state[self.PROPERTIES] = set()
|
|
self.state[self.PARENT_TYPES] = set()
|
|
self.state[self.RELATIONSHIPS] = set()
|
|
self.state[self.CONNECTED] = set()
|
|
self.state[self.ACTIONS] = dict()
|
|
|
|
# Workaround for 401 error issue
|
|
try:
|
|
# Moves _translate_packages above translate_services to
|
|
# make use of properties table in translate_services
|
|
logger.debug("Murano grabbing packages")
|
|
packages = self.murano_client.packages.list()
|
|
self._translate_packages(packages)
|
|
|
|
logger.debug("Murano grabbing environments")
|
|
environments = self.murano_client.environments.list()
|
|
self._translate_environments(environments)
|
|
self._translate_services(environments)
|
|
self._translate_deployments(environments)
|
|
self._translate_connected()
|
|
except murano_exceptions.HTTPException:
|
|
raise
|
|
|
|
@classmethod
|
|
def get_schema(cls):
|
|
"""Returns a dictionary of table schema.
|
|
|
|
The dictionary mapping tablenames to the list of column names
|
|
for that table. Both tablenames and columnnames are strings.
|
|
"""
|
|
d = {}
|
|
d[cls.OBJECTS] = ('object_id', 'owner_id', 'type')
|
|
# parent_types include not only the type of object's immediate
|
|
# parent but also all of its ancestors and its own type. The
|
|
# additional info helps writing better datalog rules.
|
|
d[cls.PARENT_TYPES] = ('id', 'parent_type')
|
|
d[cls.PROPERTIES] = ('owner_id', 'name', 'value')
|
|
d[cls.RELATIONSHIPS] = ('source_id', 'target_id', 'name')
|
|
d[cls.CONNECTED] = ('source_id', 'target_id')
|
|
d[cls.STATES] = ('id', 'state')
|
|
return d
|
|
|
|
def _translate_environments(self, environments):
|
|
"""Translate the environments into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from environments
|
|
"""
|
|
logger.debug("_translate_environments: %s", environments)
|
|
if not environments:
|
|
return
|
|
self.state[self.STATES] = set()
|
|
if self.OBJECTS not in self.state:
|
|
self.state[self.OBJECTS] = set()
|
|
if self.PROPERTIES not in self.state:
|
|
self.state[self.PROPERTIES] = set()
|
|
if self.PARENT_TYPES not in self.state:
|
|
self.state[self.PARENT_TYPES] = set()
|
|
if self.RELATIONSHIPS not in self.state:
|
|
self.state[self.RELATIONSHIPS] = set()
|
|
if self.CONNECTED not in self.state:
|
|
self.state[self.CONNECTED] = set()
|
|
|
|
env_type = 'io.murano.Environment'
|
|
for env in environments:
|
|
self.state[self.OBJECTS].add(
|
|
(env.id, env.tenant_id, env_type))
|
|
self.state[self.STATES].add((env.id, env.status))
|
|
parent_types = self._get_parent_types(env_type)
|
|
self._add_parent_types(env.id, parent_types)
|
|
for key, value in env.to_dict().items():
|
|
if key in self.UNUSED_ENV_PROPERTIES:
|
|
continue
|
|
self._add_properties(env.id, key, value)
|
|
|
|
def _translate_services(self, environments):
|
|
"""Translate the environment services into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from services
|
|
"""
|
|
logger.debug("Murano grabbing environments services")
|
|
if not environments:
|
|
return
|
|
for env in environments:
|
|
services = self.murano_client.services.list(env.id)
|
|
self._translate_environment_services(services, env.id)
|
|
|
|
def _translate_environment_services(self, services, env_id):
|
|
"""Translate the environment services into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from services
|
|
"""
|
|
|
|
# clean actions for given environment
|
|
if self.ACTIONS not in self.state:
|
|
self.state[self.ACTIONS] = dict()
|
|
env_actions = self.state[self.ACTIONS][env_id] = set()
|
|
|
|
if not services:
|
|
return
|
|
for s in services:
|
|
s_dict = s.to_dict()
|
|
s_id = s_dict['?']['id']
|
|
s_type = s_dict['?']['type']
|
|
self.state[self.OBJECTS].add((s_id, env_id, s_type))
|
|
for key, value in s_dict.items():
|
|
if key in ['instance', '?']:
|
|
continue
|
|
self._add_properties(s_id, key, value)
|
|
self._add_relationships(s_id, key, value)
|
|
|
|
parent_types = self._get_parent_types(s_type)
|
|
self._add_parent_types(s_id, parent_types)
|
|
self._add_relationships(env_id, 'services', s_id)
|
|
self._translate_service_action(s_dict, env_actions)
|
|
|
|
if 'instance' not in s_dict:
|
|
continue
|
|
# populate service instance
|
|
si_dict = s.instance
|
|
si_id = si_dict['?']['id']
|
|
si_type = si_dict['?']['type']
|
|
self.state[self.OBJECTS].add((si_id, s_id, si_type))
|
|
|
|
for key, value in si_dict.items():
|
|
if key in ['?']:
|
|
continue
|
|
self._add_properties(si_id, key, value)
|
|
if key not in ['image']:
|
|
# there's no murano image object in the environment,
|
|
# therefore glance 'image' relationship is irrelevant
|
|
# at this point.
|
|
self._add_relationships(si_id, key, value)
|
|
# There's a relationship between the service and instance
|
|
self._add_relationships(s_id, 'instance', si_id)
|
|
|
|
parent_types = self._get_parent_types(si_type)
|
|
self._add_parent_types(si_id, parent_types)
|
|
self._translate_service_action(si_dict, env_actions)
|
|
|
|
def _translate_service_action(self, obj_dict, env_actions):
|
|
"""Translates environment's object actions to env_actions structure.
|
|
|
|
env_actions: [(obj_id, action_id, action_name, enabled)]
|
|
:param: obj_dict: object dictionary
|
|
:param: env_actions: set of environment actions
|
|
"""
|
|
obj_id = obj_dict['?']['id']
|
|
if '_actions' in obj_dict['?']:
|
|
o_actions = obj_dict['?']['_actions']
|
|
if not o_actions:
|
|
return
|
|
for action_id, action_value in o_actions.items():
|
|
action_name = action_value.get('name', '')
|
|
enabled = action_value.get('enabled', False)
|
|
action = (obj_id, action_id, action_name, enabled)
|
|
env_actions.add(action)
|
|
# TODO(tranldt): support action arguments.
|
|
# If action arguments are included in '_actions',
|
|
# they can be populated into tables.
|
|
|
|
def _translate_deployments(self, environments):
|
|
"""Translate the environment deployments into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from deployments
|
|
"""
|
|
if not environments:
|
|
return
|
|
for env in environments:
|
|
deployments = self.murano_client.deployments.list(env.id)
|
|
self._translate_environment_deployments(deployments, env.id)
|
|
|
|
def _translate_environment_deployments(self, deployments, env_id):
|
|
"""Translate the environment deployments into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from deployments
|
|
"""
|
|
if not deployments:
|
|
return
|
|
for d in deployments:
|
|
if 'defaultNetworks' not in d.description:
|
|
continue
|
|
default_networks = d.description['defaultNetworks']
|
|
net_id = None
|
|
if 'environment' in default_networks:
|
|
net_id = default_networks['environment']['?']['id']
|
|
net_type = default_networks['environment']['?']['type']
|
|
self.state[self.OBJECTS].add((net_id, env_id, net_type))
|
|
|
|
parent_types = self._get_parent_types(net_type)
|
|
self._add_parent_types(net_id, parent_types)
|
|
|
|
for key, value in default_networks['environment'].items():
|
|
if key in ['?']:
|
|
continue
|
|
self._add_properties(net_id, key, value)
|
|
|
|
if not net_id:
|
|
continue
|
|
self._add_relationships(env_id, 'defaultNetworks', net_id)
|
|
for key, value in default_networks.items():
|
|
if key in ['environment']:
|
|
# data from environment already populated
|
|
continue
|
|
new_key = 'defaultNetworks.' + key
|
|
self._add_properties(net_id, new_key, value)
|
|
# services from deployment are not of interest because the same
|
|
# info is obtained from services API
|
|
|
|
def _translate_packages(self, packages):
|
|
"""Translate the packages into tables.
|
|
|
|
Assigns self.state[tablename] for all those TABLENAMEs
|
|
generated from packages/applications
|
|
"""
|
|
# packages is a generator type
|
|
if not packages:
|
|
return
|
|
if self.OBJECTS not in self.state:
|
|
self.state[self.OBJECTS] = set()
|
|
if self.PROPERTIES not in self.state:
|
|
self.state[self.PROPERTIES] = set()
|
|
|
|
for pkg in packages:
|
|
logger.debug("pkg=%s", pkg.to_dict())
|
|
pkg_type = pkg.type
|
|
if pkg.type == 'Application':
|
|
pkg_type = 'io.murano.Application'
|
|
self.state[self.OBJECTS].add((pkg.id, pkg.owner_id, pkg_type))
|
|
|
|
for key, value in pkg.to_dict().items():
|
|
if key in self.UNUSED_PKG_PROPERTIES:
|
|
continue
|
|
self._add_properties(pkg.id, key, value)
|
|
|
|
def _add_properties(self, obj_id, key, value):
|
|
"""Add a set of (obj_id, key, value) to properties table.
|
|
|
|
:param: obj_id: uuid of object
|
|
:param: key: property name. For the case value is a list, the
|
|
same key is used for multiple values.
|
|
:param: value: property value. If value is a dict, the nested
|
|
properties will be mapped using dot notation.
|
|
"""
|
|
if value is None or value == '':
|
|
return
|
|
if isinstance(value, dict):
|
|
for k, v in value.items():
|
|
new_key = key + "." + k
|
|
self._add_properties(obj_id, new_key, v)
|
|
elif isinstance(value, list):
|
|
if len(value) == 0:
|
|
return
|
|
for item in value:
|
|
self.state[self.PROPERTIES].add(
|
|
(obj_id, key, item))
|
|
else:
|
|
self.state[self.PROPERTIES].add(
|
|
(obj_id, key, value))
|
|
|
|
def _add_relationships(self, obj_id, key, value):
|
|
"""Add a set of (obj_id, value, key) to relationships table.
|
|
|
|
:param: obj_id: source uuid
|
|
:param: key: relationship name
|
|
:param: value: target uuid
|
|
"""
|
|
if (not isinstance(value, six.string_types) or
|
|
not uuidutils.is_uuid_like(value)):
|
|
return
|
|
logger.debug("Relationship: source = %s, target = %s, rel_name = %s"
|
|
% (obj_id, value, key))
|
|
self.state[self.RELATIONSHIPS].add((obj_id, value, key))
|
|
|
|
def _transitive_closure(self):
|
|
"""Computes transitive closure on a directed graph.
|
|
|
|
In other words computes reachability within the graph.
|
|
E.g. {(1, 2), (2, 3)} -> {(1, 2), (2, 3), (1, 3)}
|
|
(1, 3) was added because there is path from 1 to 3 in the graph.
|
|
"""
|
|
closure = self.state[self.CONNECTED]
|
|
while True:
|
|
# Attempts to discover new transitive relations
|
|
# by joining 2 subsequent relations/edges within the graph.
|
|
new_relations = {(x, w) for x, y in closure
|
|
for q, w in closure if q == y}
|
|
# Creates union with already discovered relations.
|
|
closure_until_now = closure | new_relations
|
|
# If no new relations were discovered in last cycle
|
|
# the computation is finished.
|
|
if closure_until_now == closure:
|
|
self.state[self.CONNECTED] = closure
|
|
break
|
|
closure = closure_until_now
|
|
|
|
def _add_connected(self, source_id, target_id):
|
|
"""Looks up the target_id in objects and add links to connected table.
|
|
|
|
Adds sets of (source_id, target_id) to connected table along
|
|
with its indirections.
|
|
:param: source_id: source uuid
|
|
:param: target_id: target uuid
|
|
"""
|
|
for row in self.state[self.OBJECTS]:
|
|
if row[1] == target_id:
|
|
self.state[self.CONNECTED].add((row[1], row[0]))
|
|
self.state[self.CONNECTED].add((source_id, row[0]))
|
|
self.state[self.CONNECTED].add((source_id, target_id))
|
|
|
|
def _translate_connected(self):
|
|
"""Translates relationships table into connected table."""
|
|
for row in self.state[self.RELATIONSHIPS]:
|
|
self._add_connected(row[0], row[1])
|
|
self._transitive_closure()
|
|
|
|
def _add_parent_types(self, obj_id, parent_types):
|
|
"""Add sets of (obj_id, parent_type) to parent_types table.
|
|
|
|
:param: obj_id: uuid of object
|
|
:param: parent_types: list of parent type string
|
|
"""
|
|
if parent_types:
|
|
for p_type in parent_types:
|
|
self.state[self.PARENT_TYPES].add((obj_id, p_type))
|
|
|
|
def _get_package_type(self, class_name):
|
|
"""Determine whether obj_type is an Application or Library.
|
|
|
|
:param: class_name: <string> service/application class name
|
|
e.g. io.murano.apps.linux.Telnet.
|
|
:return: - package type (e.g. 'Application') if found.
|
|
- None if no package type found.
|
|
"""
|
|
pkg_type = None
|
|
if self.PROPERTIES in self.state:
|
|
idx_uuid = 0
|
|
idx_value = 2
|
|
uuid = None
|
|
for row in self.state[self.PROPERTIES]:
|
|
if 'class_definitions' in row and class_name in row:
|
|
uuid = row[idx_uuid]
|
|
break
|
|
if uuid:
|
|
for row in self.state[self.PROPERTIES]:
|
|
if 'type' in row and uuid == row[idx_uuid]:
|
|
pkg_type = row[idx_value]
|
|
|
|
# If the package is removed after deployed, its properties
|
|
# are not known and so above search will fail. In that case
|
|
# let's check for class_name prefix as the last resort.
|
|
if not pkg_type:
|
|
for prefix in self.APPS_TYPE_PREFIXES:
|
|
if prefix in class_name:
|
|
pkg_type = 'Application'
|
|
break
|
|
return pkg_type
|
|
|
|
def _get_parent_types(self, obj_type):
|
|
"""Get class types of all OBJ_TYPE's parents including itself.
|
|
|
|
Look up the hierarchy of OBJ_TYPE and return types of all its
|
|
ancestor including its own type.
|
|
:param: obj_type: <string>
|
|
"""
|
|
class_types = []
|
|
p = lambda x: inspect.isclass(x)
|
|
g = inspect.getmembers(murano_classes, p)
|
|
for name, cls in g:
|
|
logger.debug("%s: %s" % (name, cls))
|
|
if (cls is murano_classes.IOMuranoApps and
|
|
self._get_package_type(obj_type) == 'Application'):
|
|
cls.name = obj_type
|
|
if 'get_parent_types' in dir(cls):
|
|
class_types = cls.get_parent_types(obj_type)
|
|
if class_types:
|
|
break
|
|
return class_types
|
|
|
|
def _call_murano_action(self, environment_id, object_id, action_name):
|
|
"""Invokes action of object in Murano environment.
|
|
|
|
:param: environment_id: uuid
|
|
:param: object_id: uuid
|
|
:param: action_name: string
|
|
"""
|
|
# get action id using object_id, env_id and action name
|
|
logger.debug("Requested Murano action invoke %s on %s in %s",
|
|
action_name, object_id, environment_id)
|
|
if (not self.state[self.ACTIONS] or
|
|
environment_id not in self.state[self.ACTIONS]):
|
|
logger.warning('Datasource "%s" found no actions for '
|
|
'environment "%s"', self.name, environment_id)
|
|
return
|
|
env_actions = self.state[self.ACTIONS][environment_id]
|
|
for env_action in env_actions:
|
|
ea_obj_id, ea_action_id, ea_action_name, ea_enabled = env_action
|
|
if (object_id == ea_obj_id and action_name == ea_action_name
|
|
and ea_enabled):
|
|
logger.debug("Invoking Murano action_id = %s, action_name %s",
|
|
ea_action_id, ea_action_name)
|
|
# TODO(tranldt): support action arguments
|
|
task_id = self.murano_client.actions.call(environment_id,
|
|
ea_action_id)
|
|
logger.debug("Murano action invoked %s - task id %s",
|
|
ea_action_id, task_id)
|
|
self.action_call_returns.append(task_id)
|
|
|
|
def execute(self, action, action_args):
|
|
"""Overwrite ExecutionDriver.execute()."""
|
|
logger.info("%s:: executing %s on %s", self.name, action, action_args)
|
|
self.action_call_returns = []
|
|
positional_args = action_args.get('positional', [])
|
|
logger.debug('Processing action execution: action = %s, '
|
|
'positional args = %s', action, positional_args)
|
|
try:
|
|
env_id = positional_args[0]
|
|
obj_id = positional_args[1]
|
|
action_name = positional_args[2]
|
|
self._call_murano_action(env_id, obj_id, action_name)
|
|
except Exception as e:
|
|
logger.exception(str(e))
|