openstack/heat-translator
Code Issues Proposed changes

Handle URLs for input templates and imports

Browse Source 
Allow input templates and imported custom types to be provided as URLs
with Heat-Translator auto-detecting the type (file vs URL), and add
necessary unit tests.

Note, since for some test cases currently the required file does not
exist on github, we temporarily use files hosted somewhere else. Once
the patch is merged a follow-on patch will be submitted to fix that
issue and use URLs of the new files that are submitted with this patch.

Change-Id: I79c07c228d9d4de22f84ef384d46e3605b9a649e
Closes-Bug: #1340748
Partially Implements: blueprint tosca-namespaces
This commit is contained in:
Vahid Hashemian 2015-10-01 13:01:24 -07:00
parent 34f2e51e0d
commit b4315feb45
6 changed files with 371 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
heat_translator.py
View File

@ -11,13 +11,13 @@
# under the License.
import logging
import logging.config
import os
import sys
from toscaparser.tosca_template import ToscaTemplate
from toscaparser.utils.gettextutils import _
from toscaparser.utils.urlutils import UrlUtils
from translator.hot.tosca_translator import TOSCATranslator
"""
@ -65,12 +65,16 @@ def main():
parsed_params = {}
if len(sys.argv) > 3:
parsed_params = parse_parameters(sys.argv[3])
if os.path.isfile(path):
heat_tpl = translate(template_type, path, parsed_params)
a_file = os.path.isfile(path)
a_url = UrlUtils.validate_url(path) if not a_file else False
if a_file or a_url:
heat_tpl = translate(template_type, path, parsed_params, a_file)
if heat_tpl:
write_output(heat_tpl)
else:
raise ValueError(_("%(path)s is not a valid file.") % {'path': path})
raise ValueError(_("The path %(path)s is not a valid file or URL.") %
{'path': path})
def parse_parameters(parameter_list):
@ -87,10 +91,10 @@ def parse_parameters(parameter_list):
return parsed_inputs
def translate(sourcetype, path, parsed_params):
def translate(sourcetype, path, parsed_params, a_file):
output = None
if sourcetype == "tosca":
tosca = ToscaTemplate(path, parsed_params)
tosca = ToscaTemplate(path, parsed_params, a_file)
translator = TOSCATranslator(tosca, parsed_params)
output = translator.translate()
return output

36
translator/common/utils.py
View File

@ -16,11 +16,11 @@ import math
import numbers
import os
import re
from toscaparser.tosca_template import ToscaTemplate
from six.moves.urllib.parse import urlparse
import yaml
from toscaparser.utils.gettextutils import _
import toscaparser.utils.yamlparser
import translator
import yaml
YAML_ORDER_PARSER = toscaparser.utils.yamlparser.simple_ordered_parse
log = logging.getLogger('tosca')
@ -213,7 +213,7 @@ class TranslationUtils(object):
'''Verify tosca translation against the given hot specification.
inputs:
tosca_file: relative path to tosca input
tosca_file: relative local path or URL to the tosca input file
hot_file: relative path to expected hot output
params: dictionary of parameter name value pairs
@ -221,19 +221,41 @@ class TranslationUtils(object):
of the given tosca_file and the given hot_file.
'''
from toscaparser.tosca_template import ToscaTemplate
from translator.hot.tosca_translator import TOSCATranslator
tosca_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)), tosca_file)
a_file = os.path.isfile(tosca_tpl)
if not a_file:
tosca_tpl = tosca_file
expected_hot_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)), hot_file)
tosca = ToscaTemplate(tosca_tpl, params)
translate = translator.hot.tosca_translator.TOSCATranslator(tosca,
params)
tosca = ToscaTemplate(tosca_tpl, params, a_file)
translate = TOSCATranslator(tosca, params)
output = translate.translate()
output_dict = toscaparser.utils.yamlparser.simple_parse(output)
expected_output_dict = YamlUtils.get_dict(expected_hot_tpl)
return CompareUtils.diff_dicts(output_dict, expected_output_dict)
class UrlUtils(object):
@staticmethod
def validate_url(path):
"""Validates whether the given path is a URL or not.
If the given path includes a scheme (http, https, ftp, ...) and a net
location (a domain name such as www.github.com) it is validated as a
URL.
"""
parsed = urlparse(path)
return bool(parsed.scheme) and bool(parsed.netloc)
def str_to_num(value):
"""Convert a string representation of a number into a numeric type."""
if isinstance(value, numbers.Number):

125
translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml Normal file
View File

@ -0,0 +1,125 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
This template was added to test an 'invalid' scenario where the input template
to heat-translator is provided as a URL and that template is referencing an
import using an absolute path. The translation of this template would work
only if it is tried locally (not via a URL link but via a file system
reference) and the referenced import exists in the path below.
imports:
- /tmp/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

120
translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml Normal file
View File

@ -0,0 +1,120 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
imports:
- https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

77
translator/tests/test_tosca_hot_translation.py
View File

@ -323,3 +323,80 @@ class ToscaHotTranslationTest(TestCase):
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_with_url_import(self):
tosca_file = '../tests/data/' \
'tosca_single_instance_wordpress_with_url_import.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_by_url_with_local_import(self):
tosca_file = 'https://raw.githubusercontent.com/openstack/' \
'heat-translator/master/translator/tests/data/' \
'tosca_single_instance_wordpress.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_by_url_with_local_abspath_import(self):
tosca_file = 'https://ibm.box.com/shared/static/' \
'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
try:
TranslationUtils.compare_tosca_translation_with_hot(
tosca_file, hot_file, params)
except Exception as err:
self.assertTrue(isinstance(err, ImportError))
self.assertEqual(
'Absolute file name /tmp/wordpress.yaml cannot be used for a '
'URL-based input https://ibm.box.com/shared/static/'
'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml template.',
err.__str__())
else:
raise Exception(
'The unit test that was expected to fail did not fail.')
def test_hot_translate_template_by_url_with_url_import(self):
tosca_url = 'https://ibm.box.com/shared/static/' \
'tocmxe9b9x7to0lj5ph9mx58d47ol77m.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_url,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))

10
translator/tests/test_utils.py
View File

@ -20,6 +20,7 @@ class CommonUtilsTest(TestCase):
MemoryUnit = translator.common.utils.MemoryUnit
cmpUtils = translator.common.utils.CompareUtils
yamlUtils = translator.common.utils.YamlUtils
UrlUtils = translator.common.utils.UrlUtils
def test_convert_unit_size_to_num(self):
size = '1 TB'
@ -227,3 +228,12 @@ class CommonUtilsTest(TestCase):
value = 1
output = translator.common.utils.str_to_num(value)
self.assertEqual(value, output)
def test_urlutils_validate_url(self):
self.assertTrue(self.UrlUtils.validate_url("http://www.github.com/"))
self.assertTrue(
self.UrlUtils.validate_url("https://github.com:81/a/2/a.b"))
self.assertTrue(self.UrlUtils.validate_url("ftp://github.com"))
self.assertFalse(self.UrlUtils.validate_url("github.com"))
self.assertFalse(self.UrlUtils.validate_url("123"))
self.assertFalse(self.UrlUtils.validate_url("a/b/c"))