openstack/heat-translator
Code Issues Proposed changes

Merge "Handle URLs for input templates and imports"

Browse Source
This commit is contained in:
Jenkins 2015-10-01 23:28:03 +00:00 committed by Gerrit Code Review
commit e6ba8b486a
6 changed files with 371 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
heat_translator.py
View File

@ -11,13 +11,13 @@
# under the License.
import logging
import logging.config
import os
import sys
from toscaparser.tosca_template import ToscaTemplate
from toscaparser.utils.gettextutils import _
from toscaparser.utils.urlutils import UrlUtils
from translator.hot.tosca_translator import TOSCATranslator
"""
@ -65,12 +65,16 @@ def main():
parsed_params = {}
if len(sys.argv) > 3:
parsed_params = parse_parameters(sys.argv[3])
if os.path.isfile(path):
heat_tpl = translate(template_type, path, parsed_params)
a_file = os.path.isfile(path)
a_url = UrlUtils.validate_url(path) if not a_file else False
if a_file or a_url:
heat_tpl = translate(template_type, path, parsed_params, a_file)
if heat_tpl:
write_output(heat_tpl)
else:
raise ValueError(_("%(path)s is not a valid file.") % {'path': path})
raise ValueError(_("The path %(path)s is not a valid file or URL.") %
{'path': path})
def parse_parameters(parameter_list):
@ -87,10 +91,10 @@ def parse_parameters(parameter_list):
return parsed_inputs
def translate(sourcetype, path, parsed_params):
def translate(sourcetype, path, parsed_params, a_file):
output = None
if sourcetype == "tosca":
tosca = ToscaTemplate(path, parsed_params)
tosca = ToscaTemplate(path, parsed_params, a_file)
translator = TOSCATranslator(tosca, parsed_params)
output = translator.translate()
return output

36
translator/common/utils.py
View File

@ -16,11 +16,11 @@ import math
import numbers
import os
import re
from toscaparser.tosca_template import ToscaTemplate
from six.moves.urllib.parse import urlparse
import yaml
from toscaparser.utils.gettextutils import _
import toscaparser.utils.yamlparser
import translator
import yaml
YAML_ORDER_PARSER = toscaparser.utils.yamlparser.simple_ordered_parse
log = logging.getLogger('tosca')
@ -213,7 +213,7 @@ class TranslationUtils(object):
'''Verify tosca translation against the given hot specification.
inputs:
tosca_file: relative path to tosca input
tosca_file: relative local path or URL to the tosca input file
hot_file: relative path to expected hot output
params: dictionary of parameter name value pairs
@ -221,19 +221,41 @@ class TranslationUtils(object):
of the given tosca_file and the given hot_file.
'''
from toscaparser.tosca_template import ToscaTemplate
from translator.hot.tosca_translator import TOSCATranslator
tosca_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)), tosca_file)
a_file = os.path.isfile(tosca_tpl)
if not a_file:
tosca_tpl = tosca_file
expected_hot_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)), hot_file)
tosca = ToscaTemplate(tosca_tpl, params)
translate = translator.hot.tosca_translator.TOSCATranslator(tosca,
params)
tosca = ToscaTemplate(tosca_tpl, params, a_file)
translate = TOSCATranslator(tosca, params)
output = translate.translate()
output_dict = toscaparser.utils.yamlparser.simple_parse(output)
expected_output_dict = YamlUtils.get_dict(expected_hot_tpl)
return CompareUtils.diff_dicts(output_dict, expected_output_dict)
class UrlUtils(object):
@staticmethod
def validate_url(path):
"""Validates whether the given path is a URL or not.
If the given path includes a scheme (http, https, ftp, ...) and a net
location (a domain name such as www.github.com) it is validated as a
URL.
"""
parsed = urlparse(path)
return bool(parsed.scheme) and bool(parsed.netloc)
def str_to_num(value):
"""Convert a string representation of a number into a numeric type."""
if isinstance(value, numbers.Number):

125
translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml Normal file
View File

@ -0,0 +1,125 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
This template was added to test an 'invalid' scenario where the input template
to heat-translator is provided as a URL and that template is referencing an
import using an absolute path. The translation of this template would work
only if it is tried locally (not via a URL link but via a file system
reference) and the referenced import exists in the path below.
imports:
- /tmp/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

120
translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml Normal file
View File

@ -0,0 +1,120 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
imports:
- https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

77
translator/tests/test_tosca_hot_translation.py
View File

@ -323,3 +323,80 @@ class ToscaHotTranslationTest(TestCase):
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_with_url_import(self):
tosca_file = '../tests/data/' \
'tosca_single_instance_wordpress_with_url_import.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_by_url_with_local_import(self):
tosca_file = 'https://raw.githubusercontent.com/openstack/' \
'heat-translator/master/translator/tests/data/' \
'tosca_single_instance_wordpress.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))
def test_hot_translate_template_by_url_with_local_abspath_import(self):
tosca_file = 'https://ibm.box.com/shared/static/' \
'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
try:
TranslationUtils.compare_tosca_translation_with_hot(
tosca_file, hot_file, params)
except Exception as err:
self.assertTrue(isinstance(err, ImportError))
self.assertEqual(
'Absolute file name /tmp/wordpress.yaml cannot be used for a '
'URL-based input https://ibm.box.com/shared/static/'
'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml template.',
err.__str__())
else:
raise Exception(
'The unit test that was expected to fail did not fail.')
def test_hot_translate_template_by_url_with_url_import(self):
tosca_url = 'https://ibm.box.com/shared/static/' \
'tocmxe9b9x7to0lj5ph9mx58d47ol77m.yaml'
hot_file = '../tests/data/hot_output/' \
'hot_single_instance_wordpress.yaml'
params = {'db_name': 'wordpress',
'db_user': 'wp_user',
'db_pwd': 'wp_pass',
'db_root_pwd': 'passw0rd',
'db_port': 3366,
'cpus': 8}
diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_url,
hot_file,
params)
self.assertEqual({}, diff, '<difference> : ' +
json.dumps(diff, indent=4, separators=(', ', ': ')))

10
translator/tests/test_utils.py
View File

@ -20,6 +20,7 @@ class CommonUtilsTest(TestCase):
MemoryUnit = translator.common.utils.MemoryUnit
cmpUtils = translator.common.utils.CompareUtils
yamlUtils = translator.common.utils.YamlUtils
UrlUtils = translator.common.utils.UrlUtils
def test_convert_unit_size_to_num(self):
size = '1 TB'
@ -227,3 +228,12 @@ class CommonUtilsTest(TestCase):
value = 1
output = translator.common.utils.str_to_num(value)
self.assertEqual(value, output)
def test_urlutils_validate_url(self):
self.assertTrue(self.UrlUtils.validate_url("http://www.github.com/"))
self.assertTrue(
self.UrlUtils.validate_url("https://github.com:81/a/2/a.b"))
self.assertTrue(self.UrlUtils.validate_url("ftp://github.com"))
self.assertFalse(self.UrlUtils.validate_url("github.com"))
self.assertFalse(self.UrlUtils.validate_url("123"))
self.assertFalse(self.UrlUtils.validate_url("a/b/c"))