openstack/heat-translator
Code Issues Proposed changes

Enable deployment of CSAR with relative path references

Browse Source 
Provide implementation to automatically deploy CSAR with relative path
references for scripts etc.

Change-Id: I376059f36dae1080c01adb24834425a481c539a6
Related-Bug: #1631060
This commit is contained in:
sahdev zala 2016-10-10 18:08:49 -04:00
parent 5b791b5115
commit cdae589c7f
20 changed files with 193 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
translator/common/utils.py
View File

@ -20,7 +20,9 @@ import re
import requests
import six
from six.moves.urllib.parse import urlparse
import tempfile
import yaml
import zipfile
from toscaparser.utils.gettextutils import _
import toscaparser.utils.yamlparser
@ -323,3 +325,30 @@ def get_token_id(access_dict):
if access_dict is None:
return None
return access_dict['access']['token']['id']
def decompress(zip_file, dir=None):
"""Decompress Zip file
Decompress any zip file. For example, TOSCA CSAR
inputs:
zip_file: file in zip format
dir: directory to decompress zip. If not provided an unique temporary
directory will be generated and used.
return:
dir: absolute path to the decopressed directory
"""
if not dir:
dir = tempfile.NamedTemporaryFile().name
with zipfile.ZipFile(zip_file, "r") as zf:
zf.extractall(dir)
return dir
def get_dict_value(dict_item, key, get_files):
if key in dict_item:
return get_files.append(dict_item[key])
for k, v in dict_item.items():
if isinstance(v, dict):
get_dict_value(v, key, get_files)

33
translator/hot/syntax/hot_resource.py
View File

@ -38,7 +38,7 @@ class HotResource(object):
def __init__(self, nodetemplate, name=None, type=None, properties=None,
metadata=None, depends_on=None,
update_policy=None, deletion_policy=None):
update_policy=None, deletion_policy=None, csar_dir=None):
log.debug(_('Translating TOSCA node type to HOT resource type.'))
self.nodetemplate = nodetemplate
if name:
@ -47,11 +47,19 @@ class HotResource(object):
self.name = nodetemplate.name
self.type = type
self.properties = properties or {}
self.csar_dir = csar_dir
# special case for HOT softwareconfig
cwd = os.getcwd()
if type == 'OS::Heat::SoftwareConfig':
config = self.properties.get('config')
if isinstance(config, dict):
implementation_artifact = config.get('get_file')
if self.csar_dir:
os.chdir(self.csar_dir)
implementation_artifact = os.path.abspath(config.get(
'get_file'))
else:
implementation_artifact = config.get('get_file')
if implementation_artifact:
filename, file_extension = os.path.splitext(
implementation_artifact)
@ -68,7 +76,7 @@ class HotResource(object):
if self.properties.get('group') is None:
self.properties['group'] = 'script'
os.chdir(cwd)
self.metadata = metadata
# The difference between depends_on and depends_on_nodes is
@ -139,16 +147,23 @@ class HotResource(object):
if hosting_on_server is None and base_type == 'tosca.nodes.Compute':
hosting_on_server = self.name
cwd = os.getcwd()
for operation in operations.values():
if operation.name in operations_deploy_sequence:
config_name = node_name + '_' + operation.name + '_config'
deploy_name = node_name + '_' + operation.name + '_deploy'
if self.csar_dir:
os.chdir(self.csar_dir)
get_file = os.path.abspath(operation.implementation)
else:
get_file = operation.implementation
hot_resources.append(
HotResource(self.nodetemplate,
config_name,
'OS::Heat::SoftwareConfig',
{'config':
{'get_file': operation.implementation}}))
{'get_file': get_file}},
csar_dir=self.csar_dir))
if operation.name == reserve_current and \
base_type != 'tosca.nodes.Compute':
deploy_resource = self
@ -166,13 +181,14 @@ class HotResource(object):
HotResource(self.nodetemplate,
deploy_name,
'OS::Heat::SoftwareDeployment',
sd_config)
sd_config, csar_dir=self.csar_dir)
hot_resources.append(deploy_resource)
deploy_lookup[operation] = deploy_resource
lifecycle_inputs = self._get_lifecycle_inputs(operation)
if lifecycle_inputs:
deploy_resource.properties['input_values'] = \
lifecycle_inputs
os.chdir(cwd)
# Add dependencies for the set of HOT resources in the sequence defined
# in operations_deploy_sequence
@ -244,14 +260,15 @@ class HotResource(object):
hot_resources.append(
HotResource(self.nodetemplate, config_name,
'OS::Heat::SoftwareConfig',
{'config': install_roles_script}))
{'config': install_roles_script},
csar_dir=self.csar_dir))
sd_config = {'config': {'get_resource': config_name},
'server': {'get_resource':
hosting_on_server}}
deploy_resource = \
HotResource(self.nodetemplate, deploy_name,
'OS::Heat::SoftwareDeployment',
sd_config)
sd_config, csar_dir=self.csar_dir)
hot_resources.append(deploy_resource)
return deploy_resource
@ -278,7 +295,7 @@ class HotResource(object):
deploy_name,
'OS::Heat::SoftwareDeployment',
sd_config,
depends_on=[hot_depends])
depends_on=[hot_depends], csar_dir=self.csar_dir)
connect_inputs = self._get_connect_inputs(config_location, operation)
if connect_inputs:
deploy_resource.properties['input_values'] = connect_inputs

5
translator/hot/tosca/tosca_block_storage.py
View File

@ -29,9 +29,10 @@ class ToscaBlockStorage(HotResource):
toscatype = 'tosca.nodes.BlockStorage'
def __init__(self, nodetemplate):
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaBlockStorage, self).__init__(nodetemplate,
type='OS::Cinder::Volume')
type='OS::Cinder::Volume',
csar_dir=csar_dir)
pass
def handle_properties(self):

6
translator/hot/tosca/tosca_block_storage_attachment.py
View File

@ -23,9 +23,11 @@ class ToscaBlockStorageAttachment(HotResource):
toscatype = 'tosca.nodes.BlockStorageAttachment'
def __init__(self, template, nodetemplates, instance_uuid, volume_id):
def __init__(self, template, nodetemplates, instance_uuid, volume_id,
csar_dir=None):
super(ToscaBlockStorageAttachment,
self).__init__(template, type='OS::Cinder::VolumeAttachment')
self).__init__(template, type='OS::Cinder::VolumeAttachment',
csar_dir=csar_dir)
self.nodetemplates = nodetemplates
self.instance_uuid = {'get_resource': instance_uuid}
self.volume_id = {'get_resource': volume_id}

5
translator/hot/tosca/tosca_cluster_policies_scaling.py
View File

@ -34,10 +34,11 @@ class ToscaClusterAutoscaling(HotResource):
toscatype = 'tosca.policies.Scaling.Cluster'
def __init__(self, policy):
def __init__(self, policy, csar_dir=None):
hot_type = "OS::Senlin::Policy"
super(ToscaClusterAutoscaling, self).__init__(policy,
type=hot_type)
type=hot_type,
csar_dir=csar_dir)
self.policy = policy
def _generate_scale_properties(self,

5
translator/hot/tosca/tosca_compute.py
View File

@ -45,9 +45,10 @@ class ToscaCompute(HotResource):
'scheduler_hints', 'security_groups', 'software_config_transport',
'user_data', 'user_data_format', 'user_data_update_policy')
def __init__(self, nodetemplate):
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaCompute, self).__init__(nodetemplate,
type='OS::Nova::Server')
type='OS::Nova::Server',
csar_dir=csar_dir)
# List with associated hot port resources with this server
self.assoc_port_resources = []
pass

4
translator/hot/tosca/tosca_database.py
View File

@ -22,8 +22,8 @@ class ToscaDatabase(HotResource):
toscatype = 'tosca.nodes.Database'
def __init__(self, nodetemplate):
super(ToscaDatabase, self).__init__(nodetemplate)
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaDatabase, self).__init__(nodetemplate, csar_dir=csar_dir)
pass
def handle_properties(self):

4
translator/hot/tosca/tosca_dbms.py
View File

@ -22,8 +22,8 @@ class ToscaDbms(HotResource):
toscatype = 'tosca.nodes.DBMS'
def __init__(self, nodetemplate):
super(ToscaDbms, self).__init__(nodetemplate)
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaDbms, self).__init__(nodetemplate, csar_dir=csar_dir)
pass
def handle_properties(self):

5
translator/hot/tosca/tosca_network_network.py
View File

@ -28,9 +28,10 @@ class ToscaNetwork(HotResource):
existing_resource_id = None
def __init__(self, nodetemplate):
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaNetwork, self).__init__(nodetemplate,
type='OS::Neutron::Net')
type='OS::Neutron::Net',
csar_dir=csar_dir)
pass
def handle_properties(self):

5
translator/hot/tosca/tosca_network_port.py
View File

@ -24,9 +24,10 @@ class ToscaNetworkPort(HotResource):
toscatype = 'tosca.nodes.network.Port'
def __init__(self, nodetemplate):
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaNetworkPort, self).__init__(nodetemplate,
type='OS::Neutron::Port')
type='OS::Neutron::Port',
csar_dir=csar_dir)
# Default order
self.order = 0
pass

5
translator/hot/tosca/tosca_object_storage.py
View File

@ -23,9 +23,10 @@ class ToscaObjectStorage(HotResource):
toscatype = 'tosca.nodes.ObjectStorage'
def __init__(self, nodetemplate):
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaObjectStorage, self).__init__(nodetemplate,
type='OS::Swift::Container')
type='OS::Swift::Container',
csar_dir=csar_dir)
pass
def handle_properties(self):

5
translator/hot/tosca/tosca_policies.py
View File

@ -22,9 +22,10 @@ class ToscaPolicies(HotResource):
toscatype = 'tosca.policies.Placement'
def __init__(self, policy):
def __init__(self, policy, csar_dir=None):
super(ToscaPolicies, self).__init__(policy,
type='OS::Nova::ServerGroup')
type='OS::Nova::ServerGroup',
csar_dir=csar_dir)
self.policy = policy
def handle_properties(self, resources):

5
translator/hot/tosca/tosca_policies_scaling.py
View File

@ -31,10 +31,11 @@ class ToscaAutoscaling(HotResource):
toscatype = 'tosca.policies.Scaling'
def __init__(self, policy):
def __init__(self, policy, csar_dir=None):
hot_type = "OS::Heat::ScalingPolicy"
super(ToscaAutoscaling, self).__init__(policy,
type=hot_type)
type=hot_type,
csar_dir=csar_dir)
self.policy = policy
def handle_expansion(self):

5
translator/hot/tosca/tosca_software_component.py
View File

@ -22,8 +22,9 @@ class ToscaSoftwareComponent(HotResource):
toscatype = 'tosca.nodes.SoftwareComponent'
def __init__(self, nodetemplate):
super(ToscaSoftwareComponent, self).__init__(nodetemplate)
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaSoftwareComponent, self).__init__(nodetemplate,
csar_dir=csar_dir)
pass
def handle_properties(self):

5
translator/hot/tosca/tosca_web_application.py
View File

@ -22,8 +22,9 @@ class ToscaWebApplication(HotResource):
toscatype = 'tosca.nodes.WebApplication'
def __init__(self, nodetemplate):
super(ToscaWebApplication, self).__init__(nodetemplate)
def __init__(self, nodetemplate, csar_dir=None):
super(ToscaWebApplication, self).__init__(nodetemplate,
csar_dir=csar_dir)
pass
def handle_properties(self):

5
translator/hot/tosca/tosca_webserver.py
View File

@ -22,8 +22,9 @@ class ToscaWebserver(HotResource):
toscatype = 'tosca.nodes.WebServer'
def __init__(self, nodetemplate):
super(ToscaWebserver, self).__init__(nodetemplate)
def __init__(self, nodetemplate, csar_dir):
super(ToscaWebserver, self).__init__(nodetemplate,
csar_dir=csar_dir)
pass
def handle_properties(self):

6
translator/hot/tosca_translator.py
View File

@ -24,12 +24,13 @@ log = logging.getLogger('heat-translator')
class TOSCATranslator(object):
'''Invokes translation methods.'''
def __init__(self, tosca, parsed_params, deploy=None):
def __init__(self, tosca, parsed_params, deploy=None, csar_dir=None):
super(TOSCATranslator, self).__init__()
self.tosca = tosca
self.hot_template = HotTemplate()
self.parsed_params = parsed_params
self.deploy = deploy
self.csar_dir = csar_dir
self.node_translator = None
log.info(_('Initialized parmaters for translation.'))
@ -38,7 +39,8 @@ class TOSCATranslator(object):
self.hot_template.description = self.tosca.description
self.hot_template.parameters = self._translate_inputs()
self.node_translator = TranslateNodeTemplates(self.tosca,
self.hot_template)
self.hot_template,
csar_dir=self.csar_dir)
self.hot_template.resources = \
self.node_translator.translate()
self.hot_template.outputs = self._translate_outputs()

33
translator/hot/translate_node_templates.py
View File

@ -146,10 +146,11 @@ HOT_SCALING_POLICY_TYPE = ["OS::Heat::AutoScalingGroup",
class TranslateNodeTemplates(object):
'''Translate TOSCA NodeTemplates to Heat Resources.'''
def __init__(self, tosca, hot_template):
def __init__(self, tosca, hot_template, csar_dir=None):
self.tosca = tosca
self.nodetemplates = self.tosca.nodetemplates
self.hot_template = hot_template
self.csar_dir = csar_dir
# list of all HOT resources generated
self.hot_resources = []
# mapping between TOSCA nodetemplate and HOT resource
@ -218,7 +219,8 @@ class TranslateNodeTemplates(object):
base_type = HotResource.get_base_type_str(node.type_definition)
if base_type not in TOSCA_TO_HOT_TYPE:
raise UnsupportedTypeError(type=_('%s') % base_type)
hot_node = TOSCA_TO_HOT_TYPE[base_type](node)
hot_node = TOSCA_TO_HOT_TYPE[base_type](node,
csar_dir=self.csar_dir)
self.hot_resources.append(hot_node)
self.hot_lookup[node] = hot_node
@ -435,9 +437,16 @@ class TranslateNodeTemplates(object):
if tosca_target:
artifacts = HotResource.get_all_artifacts(tosca_target)
if artifact_name in artifacts:
cwd = os.getcwd()
artifact = artifacts[artifact_name]
if self.csar_dir:
os.chdir(self.csar_dir)
get_file = os.path.abspath(artifact.get('file'))
else:
get_file = artifact.get('file')
if artifact.get('type', None) == 'tosca.artifacts.File':
return {'get_file': artifact.get('file')}
return {'get_file': get_file}
os.chdir(cwd)
get_input_args = None
if isinstance(param_value, GetInput):
get_input_args = param_value.args
@ -683,16 +692,30 @@ class TranslateNodeTemplates(object):
raise Exception(msg)
config_name = source_node.name + '_' + target_name + '_connect_config'
implement = connect_config.get('implementation')
cwd = os.getcwd()
if config_location == 'target':
if self.csar_dir:
os.chdir(self.csar_dir)
get_file = os.path.abspath(implement)
else:
get_file = implement
hot_config = HotResource(target_node,
config_name,
'OS::Heat::SoftwareConfig',
{'config': {'get_file': implement}})
{'config': {'get_file': get_file}},
csar_dir=self.csar_dir)
elif config_location == 'source':
if self.csar_dir:
os.chdir(self.csar_dir)
get_file = os.path.abspath(implement)
else:
get_file = implement
hot_config = HotResource(source_node,
config_name,
'OS::Heat::SoftwareConfig',
{'config': {'get_file': implement}})
{'config': {'get_file': get_file}},
csar_dir=self.csar_dir)
os.chdir(cwd)
connectsto_resources.append(hot_config)
hot_target = self._find_hot_resource_for_tosca(target_name)
hot_source = self._find_hot_resource_for_tosca(source_node.name)

36
translator/shell.py
View File

@ -12,12 +12,14 @@
import argparse
import codecs
import logging
import logging.config
import os
import sys
import uuid
import yaml
import zipfile
# NOTE(aloga): As per upstream developers requirement this needs to work
# without the clients, therefore we need to pass if we cannot import them
@ -41,6 +43,7 @@ from toscaparser.utils.gettextutils import _
from toscaparser.utils.urlutils import UrlUtils
from translator.common import flavors
from translator.common import images
from translator.common import utils
from translator.conf.config import ConfigProvider
from translator.hot.tosca_translator import TOSCATranslator
@ -68,6 +71,7 @@ log = logging.getLogger("heat-translator")
class TranslatorShell(object):
SUPPORTED_TYPES = ['tosca']
TOSCA_CSAR_META_DIR = "TOSCA-Metadata"
def get_parser(self, argv):
parser = argparse.ArgumentParser(prog="heat-translator")
@ -207,17 +211,31 @@ class TranslatorShell(object):
% {'name': heat_stack_name}
log.debug(msg)
tpl = yaml.load(template)
# get all the values for get_file from a translated template
get_files = []
utils.get_dict_value(tpl, "get_file", get_files)
files = {}
if get_files:
for file in get_files:
with codecs.open(file, encoding='utf-8', errors='strict') \
as f:
text = f.read()
files[file] = text
tpl['heat_template_version'] = str(tpl['heat_template_version'])
self._create_stack(heat_client=heat_client,
stack_name=heat_stack_name,
template=tpl,
parameters=parameters)
parameters=parameters,
files=files)
def _create_stack(self, heat_client, stack_name, template, parameters):
def _create_stack(self, heat_client, stack_name, template, parameters,
files):
if heat_client:
heat_client.stacks.create(stack_name=stack_name,
template=template,
parameters=parameters)
parameters=parameters,
files=files)
def _parse_parameters(self, parameter_list):
parsed_inputs = {}
@ -248,7 +266,17 @@ class TranslatorShell(object):
if sourcetype == "tosca":
log.debug(_('Loading the tosca template.'))
tosca = ToscaTemplate(path, parsed_params, a_file)
translator = TOSCATranslator(tosca, parsed_params, deploy)
csar_dir = None
if deploy and zipfile.is_zipfile(path):
# set CSAR directory to the root of TOSCA-Metadata
csar_decompress = utils.decompress(path)
csar_dir = os.path.join(csar_decompress,
self.TOSCA_CSAR_META_DIR)
msg = _("'%(csar)s' is the location of decompressed "
"CSAR file.") % {'csar': csar_dir}
log.info(msg)
translator = TOSCATranslator(tosca, parsed_params, deploy,
csar_dir=csar_dir)
log.debug(_('Translating the tosca template.'))
output = translator.translate()
return output

34
translator/tests/test_utils.py
View File

@ -234,3 +234,37 @@ class CommonUtilsTest(TestCase):
self.assertFalse(self.UrlUtils.validate_url("github.com"))
self.assertFalse(self.UrlUtils.validate_url("123"))
self.assertFalse(self.UrlUtils.validate_url("a/b/c"))
def test_get_dict_value(self):
single_snippet = \
{'nodejs_create_config':
{'type': 'tosca.nodes.SoftwareConfig',
'properties':
{'config':
{'get_file': 'create.sh'}}}}
actual_output_single_snippet = []
ex_output_single_snippet = ['create.sh']
translator.common.utils.get_dict_value(single_snippet, "get_file",
actual_output_single_snippet)
self.assertEqual(actual_output_single_snippet,
ex_output_single_snippet)
multi_snippet = \
{'resources':
{'nodejs_create_config':
{'type': 'tosca.nodes.SoftwareConfig',
'properties':
{'config':
{'get_file': 'nodejs/create.sh'}}},
'mongodb_create_config':
{'type': 'tosca.nodes.SoftwareConfig',
'properties':
{'config':
{'get_file': 'mongodb/create.sh'}}}}}
actual_output_multi_snippet = []
ex_output_multi_snippet = ['mongodb/create.sh',
'nodejs/create.sh']
translator.common.utils.get_dict_value(multi_snippet, "get_file",
actual_output_multi_snippet)
self.assertEqual(sorted(actual_output_multi_snippet),
ex_output_multi_snippet)