diff --git a/heat_translator.py b/heat_translator.py index 07890260..dcb6712a 100644 --- a/heat_translator.py +++ b/heat_translator.py @@ -11,13 +11,13 @@ # under the License. -import logging import logging.config import os import sys from toscaparser.tosca_template import ToscaTemplate from toscaparser.utils.gettextutils import _ +from toscaparser.utils.urlutils import UrlUtils from translator.hot.tosca_translator import TOSCATranslator """ @@ -65,12 +65,16 @@ def main(): parsed_params = {} if len(sys.argv) > 3: parsed_params = parse_parameters(sys.argv[3]) - if os.path.isfile(path): - heat_tpl = translate(template_type, path, parsed_params) + + a_file = os.path.isfile(path) + a_url = UrlUtils.validate_url(path) if not a_file else False + if a_file or a_url: + heat_tpl = translate(template_type, path, parsed_params, a_file) if heat_tpl: write_output(heat_tpl) else: - raise ValueError(_("%(path)s is not a valid file.") % {'path': path}) + raise ValueError(_("The path %(path)s is not a valid file or URL.") % + {'path': path}) def parse_parameters(parameter_list): @@ -87,10 +91,10 @@ def parse_parameters(parameter_list): return parsed_inputs -def translate(sourcetype, path, parsed_params): +def translate(sourcetype, path, parsed_params, a_file): output = None if sourcetype == "tosca": - tosca = ToscaTemplate(path, parsed_params) + tosca = ToscaTemplate(path, parsed_params, a_file) translator = TOSCATranslator(tosca, parsed_params) output = translator.translate() return output diff --git a/translator/common/utils.py b/translator/common/utils.py index 031c3046..502b206e 100644 --- a/translator/common/utils.py +++ b/translator/common/utils.py @@ -16,11 +16,11 @@ import math import numbers import os import re -from toscaparser.tosca_template import ToscaTemplate +from six.moves.urllib.parse import urlparse +import yaml + from toscaparser.utils.gettextutils import _ import toscaparser.utils.yamlparser -import translator -import yaml YAML_ORDER_PARSER = toscaparser.utils.yamlparser.simple_ordered_parse log = logging.getLogger('tosca') @@ -213,7 +213,7 @@ class TranslationUtils(object): '''Verify tosca translation against the given hot specification. inputs: - tosca_file: relative path to tosca input + tosca_file: relative local path or URL to the tosca input file hot_file: relative path to expected hot output params: dictionary of parameter name value pairs @@ -221,19 +221,41 @@ class TranslationUtils(object): of the given tosca_file and the given hot_file. ''' + from toscaparser.tosca_template import ToscaTemplate + from translator.hot.tosca_translator import TOSCATranslator + tosca_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), tosca_file) + a_file = os.path.isfile(tosca_tpl) + if not a_file: + tosca_tpl = tosca_file + expected_hot_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), hot_file) - tosca = ToscaTemplate(tosca_tpl, params) - translate = translator.hot.tosca_translator.TOSCATranslator(tosca, - params) + + tosca = ToscaTemplate(tosca_tpl, params, a_file) + translate = TOSCATranslator(tosca, params) + output = translate.translate() output_dict = toscaparser.utils.yamlparser.simple_parse(output) expected_output_dict = YamlUtils.get_dict(expected_hot_tpl) return CompareUtils.diff_dicts(output_dict, expected_output_dict) +class UrlUtils(object): + + @staticmethod + def validate_url(path): + """Validates whether the given path is a URL or not. + + If the given path includes a scheme (http, https, ftp, ...) and a net + location (a domain name such as www.github.com) it is validated as a + URL. + """ + parsed = urlparse(path) + return bool(parsed.scheme) and bool(parsed.netloc) + + def str_to_num(value): """Convert a string representation of a number into a numeric type.""" if isinstance(value, numbers.Number): diff --git a/translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml b/translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml new file mode 100644 index 00000000..061f5aeb --- /dev/null +++ b/translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml @@ -0,0 +1,125 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with wordpress, web server and mysql on the same server. + This template was added to test an 'invalid' scenario where the input template + to heat-translator is provided as a URL and that template is referencing an + import using an absolute path. The translation of this template would work + only if it is tried locally (not via a URL link but via a file system + reference) and the referenced import exists in the path below. + +imports: + - /tmp/wordpress.yaml + +topology_template: + inputs: + cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + default: 1 + db_name: + type: string + description: The name of the database. + default: wordpress + db_user: + type: string + description: The user name of the DB user. + default: wp_user + db_pwd: + type: string + description: The WordPress database admin account password. + default: wp_pass + db_root_pwd: + type: string + description: Root password for MySQL. + db_port: + type: PortDef + description: Port for the MySQL database. + default: 3306 + + node_templates: + wordpress: + type: tosca.nodes.WebApplication.WordPress + requirements: + - host: webserver + - database_endpoint: mysql_database + interfaces: + Standard: + create: wordpress/wordpress_install.sh + configure: + implementation: wordpress/wordpress_configure.sh + inputs: + wp_db_name: wordpress + wp_db_user: wp_user + wp_db_password: wp_pass + + mysql_database: + type: tosca.nodes.Database + properties: + name: { get_input: db_name } + user: { get_input: db_user } + password: { get_input: db_pwd } + capabilities: + database_endpoint: + properties: + port: { get_input: db_port } + requirements: + - host: + node: mysql_dbms + interfaces: + Standard: + configure: + implementation: mysql/mysql_database_configure.sh + inputs: + db_name: wordpress + db_user: wp_user + db_password: wp_pass + db_root_password: passw0rd + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: { get_input: db_root_pwd } + port: { get_input: db_port } + requirements: + - host: server + interfaces: + Standard: + create: + implementation: mysql/mysql_dbms_install.sh + inputs: + db_root_password: passw0rd + start: mysql/mysql_dbms_start.sh + configure: + implementation: mysql/mysql_dbms_configure.sh + inputs: + db_port: 3366 + + webserver: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: webserver/webserver_install.sh + start: webserver/webserver_start.sh + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: cpus } + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + outputs: + website_url: + description: URL for Wordpress wiki. + value: { get_attribute: [server, private_address] } diff --git a/translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml b/translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml new file mode 100644 index 00000000..e5f1580b --- /dev/null +++ b/translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml @@ -0,0 +1,120 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with wordpress, web server and mysql on the same server. + +imports: + - https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml + +topology_template: + inputs: + cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + default: 1 + db_name: + type: string + description: The name of the database. + default: wordpress + db_user: + type: string + description: The user name of the DB user. + default: wp_user + db_pwd: + type: string + description: The WordPress database admin account password. + default: wp_pass + db_root_pwd: + type: string + description: Root password for MySQL. + db_port: + type: PortDef + description: Port for the MySQL database. + default: 3306 + + node_templates: + wordpress: + type: tosca.nodes.WebApplication.WordPress + requirements: + - host: webserver + - database_endpoint: mysql_database + interfaces: + Standard: + create: wordpress/wordpress_install.sh + configure: + implementation: wordpress/wordpress_configure.sh + inputs: + wp_db_name: wordpress + wp_db_user: wp_user + wp_db_password: wp_pass + + mysql_database: + type: tosca.nodes.Database + properties: + name: { get_input: db_name } + user: { get_input: db_user } + password: { get_input: db_pwd } + capabilities: + database_endpoint: + properties: + port: { get_input: db_port } + requirements: + - host: + node: mysql_dbms + interfaces: + Standard: + configure: + implementation: mysql/mysql_database_configure.sh + inputs: + db_name: wordpress + db_user: wp_user + db_password: wp_pass + db_root_password: passw0rd + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: { get_input: db_root_pwd } + port: { get_input: db_port } + requirements: + - host: server + interfaces: + Standard: + create: + implementation: mysql/mysql_dbms_install.sh + inputs: + db_root_password: passw0rd + start: mysql/mysql_dbms_start.sh + configure: + implementation: mysql/mysql_dbms_configure.sh + inputs: + db_port: 3366 + + webserver: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: webserver/webserver_install.sh + start: webserver/webserver_start.sh + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: cpus } + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + outputs: + website_url: + description: URL for Wordpress wiki. + value: { get_attribute: [server, private_address] } diff --git a/translator/tests/test_tosca_hot_translation.py b/translator/tests/test_tosca_hot_translation.py index b4622f72..294937c8 100644 --- a/translator/tests/test_tosca_hot_translation.py +++ b/translator/tests/test_tosca_hot_translation.py @@ -323,3 +323,80 @@ class ToscaHotTranslationTest(TestCase): params) self.assertEqual({}, diff, ' : ' + json.dumps(diff, indent=4, separators=(', ', ': '))) + + def test_hot_translate_template_with_url_import(self): + tosca_file = '../tests/data/' \ + 'tosca_single_instance_wordpress_with_url_import.yaml' + hot_file = '../tests/data/hot_output/' \ + 'hot_single_instance_wordpress.yaml' + params = {'db_name': 'wordpress', + 'db_user': 'wp_user', + 'db_pwd': 'wp_pass', + 'db_root_pwd': 'passw0rd', + 'db_port': 3366, + 'cpus': 8} + diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, + hot_file, + params) + self.assertEqual({}, diff, ' : ' + + json.dumps(diff, indent=4, separators=(', ', ': '))) + + def test_hot_translate_template_by_url_with_local_import(self): + tosca_file = 'https://raw.githubusercontent.com/openstack/' \ + 'heat-translator/master/translator/tests/data/' \ + 'tosca_single_instance_wordpress.yaml' + hot_file = '../tests/data/hot_output/' \ + 'hot_single_instance_wordpress.yaml' + params = {'db_name': 'wordpress', + 'db_user': 'wp_user', + 'db_pwd': 'wp_pass', + 'db_root_pwd': 'passw0rd', + 'db_port': 3366, + 'cpus': 8} + diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, + hot_file, + params) + self.assertEqual({}, diff, ' : ' + + json.dumps(diff, indent=4, separators=(', ', ': '))) + + def test_hot_translate_template_by_url_with_local_abspath_import(self): + tosca_file = 'https://ibm.box.com/shared/static/' \ + 'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml' + hot_file = '../tests/data/hot_output/' \ + 'hot_single_instance_wordpress.yaml' + params = {'db_name': 'wordpress', + 'db_user': 'wp_user', + 'db_pwd': 'wp_pass', + 'db_root_pwd': 'passw0rd', + 'db_port': 3366, + 'cpus': 8} + try: + TranslationUtils.compare_tosca_translation_with_hot( + tosca_file, hot_file, params) + except Exception as err: + self.assertTrue(isinstance(err, ImportError)) + self.assertEqual( + 'Absolute file name /tmp/wordpress.yaml cannot be used for a ' + 'URL-based input https://ibm.box.com/shared/static/' + 'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml template.', + err.__str__()) + else: + raise Exception( + 'The unit test that was expected to fail did not fail.') + + def test_hot_translate_template_by_url_with_url_import(self): + tosca_url = 'https://ibm.box.com/shared/static/' \ + 'tocmxe9b9x7to0lj5ph9mx58d47ol77m.yaml' + hot_file = '../tests/data/hot_output/' \ + 'hot_single_instance_wordpress.yaml' + params = {'db_name': 'wordpress', + 'db_user': 'wp_user', + 'db_pwd': 'wp_pass', + 'db_root_pwd': 'passw0rd', + 'db_port': 3366, + 'cpus': 8} + diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_url, + hot_file, + params) + self.assertEqual({}, diff, ' : ' + + json.dumps(diff, indent=4, separators=(', ', ': '))) diff --git a/translator/tests/test_utils.py b/translator/tests/test_utils.py index 84dc3ed7..59c86373 100644 --- a/translator/tests/test_utils.py +++ b/translator/tests/test_utils.py @@ -20,6 +20,7 @@ class CommonUtilsTest(TestCase): MemoryUnit = translator.common.utils.MemoryUnit cmpUtils = translator.common.utils.CompareUtils yamlUtils = translator.common.utils.YamlUtils + UrlUtils = translator.common.utils.UrlUtils def test_convert_unit_size_to_num(self): size = '1 TB' @@ -227,3 +228,12 @@ class CommonUtilsTest(TestCase): value = 1 output = translator.common.utils.str_to_num(value) self.assertEqual(value, output) + + def test_urlutils_validate_url(self): + self.assertTrue(self.UrlUtils.validate_url("http://www.github.com/")) + self.assertTrue( + self.UrlUtils.validate_url("https://github.com:81/a/2/a.b")) + self.assertTrue(self.UrlUtils.validate_url("ftp://github.com")) + self.assertFalse(self.UrlUtils.validate_url("github.com")) + self.assertFalse(self.UrlUtils.validate_url("123")) + self.assertFalse(self.UrlUtils.validate_url("a/b/c"))