Merge "Handle URLs for input templates and imports"

heat_translator.py

@@ -11,13 +11,13 @@
 #    under the License.
 
 
-import logging
 import logging.config
 import os
 import sys
 
 from toscaparser.tosca_template import ToscaTemplate
 from toscaparser.utils.gettextutils import _
+from toscaparser.utils.urlutils import UrlUtils
 from translator.hot.tosca_translator import TOSCATranslator
 
 """
@@ -65,12 +65,16 @@ def main():
     parsed_params = {}
     if len(sys.argv) > 3:
         parsed_params = parse_parameters(sys.argv[3])
-    if os.path.isfile(path):
+
-        heat_tpl = translate(template_type, path, parsed_params)
+    a_file = os.path.isfile(path)
+    a_url = UrlUtils.validate_url(path) if not a_file else False
+    if a_file or a_url:
+        heat_tpl = translate(template_type, path, parsed_params, a_file)
         if heat_tpl:
             write_output(heat_tpl)
     else:
-        raise ValueError(_("%(path)s is not a valid file.") % {'path': path})
+        raise ValueError(_("The path %(path)s is not a valid file or URL.") %
+                         {'path': path})
 
 
 def parse_parameters(parameter_list):
@@ -87,10 +91,10 @@ def parse_parameters(parameter_list):
     return parsed_inputs
 
 
-def translate(sourcetype, path, parsed_params):
+def translate(sourcetype, path, parsed_params, a_file):
     output = None
     if sourcetype == "tosca":
-        tosca = ToscaTemplate(path, parsed_params)
+        tosca = ToscaTemplate(path, parsed_params, a_file)
         translator = TOSCATranslator(tosca, parsed_params)
         output = translator.translate()
     return output

translator/common/utils.py

@@ -16,11 +16,11 @@ import math
 import numbers
 import os
 import re
-from toscaparser.tosca_template import ToscaTemplate
+from six.moves.urllib.parse import urlparse
+import yaml
+
 from toscaparser.utils.gettextutils import _
 import toscaparser.utils.yamlparser
-import translator
-import yaml
 
 YAML_ORDER_PARSER = toscaparser.utils.yamlparser.simple_ordered_parse
 log = logging.getLogger('tosca')
@@ -213,7 +213,7 @@ class TranslationUtils(object):
         '''Verify tosca translation against the given hot specification.
 
         inputs:
-        tosca_file: relative path to tosca input
+        tosca_file: relative local path or URL to the tosca input file
         hot_file: relative path to expected hot output
         params: dictionary of parameter name value pairs
 
@@ -221,19 +221,41 @@ class TranslationUtils(object):
         of the given tosca_file and the given hot_file.
         '''
 
+        from toscaparser.tosca_template import ToscaTemplate
+        from translator.hot.tosca_translator import TOSCATranslator
+
         tosca_tpl = os.path.join(
             os.path.dirname(os.path.abspath(__file__)), tosca_file)
+        a_file = os.path.isfile(tosca_tpl)
+        if not a_file:
+            tosca_tpl = tosca_file
+
         expected_hot_tpl = os.path.join(
             os.path.dirname(os.path.abspath(__file__)), hot_file)
-        tosca = ToscaTemplate(tosca_tpl, params)
+
-        translate = translator.hot.tosca_translator.TOSCATranslator(tosca,
+        tosca = ToscaTemplate(tosca_tpl, params, a_file)
-                                                                    params)
+        translate = TOSCATranslator(tosca, params)
+
         output = translate.translate()
         output_dict = toscaparser.utils.yamlparser.simple_parse(output)
         expected_output_dict = YamlUtils.get_dict(expected_hot_tpl)
         return CompareUtils.diff_dicts(output_dict, expected_output_dict)
 
 
+class UrlUtils(object):
+
+    @staticmethod
+    def validate_url(path):
+        """Validates whether the given path is a URL or not.
+
+        If the given path includes a scheme (http, https, ftp, ...) and a net
+        location (a domain name such as www.github.com) it is validated as a
+        URL.
+        """
+        parsed = urlparse(path)
+        return bool(parsed.scheme) and bool(parsed.netloc)
+
+
 def str_to_num(value):
     """Convert a string representation of a number into a numeric type."""
     if isinstance(value, numbers.Number):

translator/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml

@@ -0,0 +1,125 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: >
+  TOSCA simple profile with wordpress, web server and mysql on the same server.
+  This template was added to test an 'invalid' scenario where the input template
+  to heat-translator is provided as a URL and that template is referencing an
+  import using an absolute path. The translation of this template would work
+  only if it is tried locally (not via a URL link but via a file system
+  reference) and the referenced import exists in the path below.
+
+imports:
+  - /tmp/wordpress.yaml
+
+topology_template:
+  inputs:
+    cpus:
+      type: integer
+      description: Number of CPUs for the server.
+      constraints:
+        - valid_values: [ 1, 2, 4, 8 ]
+      default: 1
+    db_name:
+      type: string
+      description: The name of the database.
+      default: wordpress
+    db_user:
+      type: string
+      description: The user name of the DB user.
+      default: wp_user
+    db_pwd:
+      type: string
+      description: The WordPress database admin account password.
+      default: wp_pass
+    db_root_pwd:
+      type: string
+      description: Root password for MySQL.
+    db_port:
+      type: PortDef
+      description: Port for the MySQL database.
+      default: 3306
+
+  node_templates:
+    wordpress:
+      type: tosca.nodes.WebApplication.WordPress
+      requirements:
+        - host: webserver
+        - database_endpoint: mysql_database
+      interfaces:
+        Standard:
+          create: wordpress/wordpress_install.sh
+          configure:
+            implementation: wordpress/wordpress_configure.sh
+            inputs:
+              wp_db_name: wordpress
+              wp_db_user: wp_user
+              wp_db_password: wp_pass
+
+    mysql_database:
+      type: tosca.nodes.Database
+      properties:
+        name: { get_input: db_name }
+        user: { get_input: db_user }
+        password: { get_input: db_pwd }
+      capabilities:
+        database_endpoint:
+          properties:
+            port: { get_input: db_port }
+      requirements:
+        - host:
+            node: mysql_dbms
+      interfaces:
+        Standard:
+          configure:
+            implementation: mysql/mysql_database_configure.sh
+            inputs:
+              db_name: wordpress
+              db_user: wp_user
+              db_password: wp_pass
+              db_root_password: passw0rd
+    mysql_dbms:
+      type: tosca.nodes.DBMS
+      properties:
+        root_password: { get_input: db_root_pwd }
+        port: { get_input: db_port }
+      requirements:
+        - host: server
+      interfaces:
+        Standard:
+          create:
+            implementation: mysql/mysql_dbms_install.sh
+            inputs:
+              db_root_password: passw0rd
+          start: mysql/mysql_dbms_start.sh
+          configure:
+            implementation: mysql/mysql_dbms_configure.sh
+            inputs:
+              db_port: 3366
+
+    webserver:
+      type: tosca.nodes.WebServer
+      requirements:
+        - host: server
+      interfaces:
+        Standard:
+          create: webserver/webserver_install.sh
+          start: webserver/webserver_start.sh
+    server:
+      type: tosca.nodes.Compute
+      capabilities:
+        host:
+          properties:
+            disk_size: 10 GB
+            num_cpus: { get_input: cpus }
+            mem_size: 4096 MB
+        os:
+          properties:
+            architecture: x86_64
+            type: Linux
+            distribution: Ubuntu
+            version: 14.04
+
+  outputs:
+    website_url:
+      description: URL for Wordpress wiki.
+      value: { get_attribute: [server, private_address] }

translator/tests/data/tosca_single_instance_wordpress_with_url_import.yaml

@@ -0,0 +1,120 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: >
+  TOSCA simple profile with wordpress, web server and mysql on the same server.
+
+imports:
+  - https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml
+
+topology_template:
+  inputs:
+    cpus:
+      type: integer
+      description: Number of CPUs for the server.
+      constraints:
+        - valid_values: [ 1, 2, 4, 8 ]
+      default: 1
+    db_name:
+      type: string
+      description: The name of the database.
+      default: wordpress
+    db_user:
+      type: string
+      description: The user name of the DB user.
+      default: wp_user
+    db_pwd:
+      type: string
+      description: The WordPress database admin account password.
+      default: wp_pass
+    db_root_pwd:
+      type: string
+      description: Root password for MySQL.
+    db_port:
+      type: PortDef
+      description: Port for the MySQL database.
+      default: 3306
+
+  node_templates:
+    wordpress:
+      type: tosca.nodes.WebApplication.WordPress
+      requirements:
+        - host: webserver
+        - database_endpoint: mysql_database
+      interfaces:
+        Standard:
+          create: wordpress/wordpress_install.sh
+          configure:
+            implementation: wordpress/wordpress_configure.sh
+            inputs:
+              wp_db_name: wordpress
+              wp_db_user: wp_user
+              wp_db_password: wp_pass
+
+    mysql_database:
+      type: tosca.nodes.Database
+      properties:
+        name: { get_input: db_name }
+        user: { get_input: db_user }
+        password: { get_input: db_pwd }
+      capabilities:
+        database_endpoint:
+          properties:
+            port: { get_input: db_port }
+      requirements:
+        - host:
+            node: mysql_dbms
+      interfaces:
+        Standard:
+          configure:
+            implementation: mysql/mysql_database_configure.sh
+            inputs:
+              db_name: wordpress
+              db_user: wp_user
+              db_password: wp_pass
+              db_root_password: passw0rd
+    mysql_dbms:
+      type: tosca.nodes.DBMS
+      properties:
+        root_password: { get_input: db_root_pwd }
+        port: { get_input: db_port }
+      requirements:
+        - host: server
+      interfaces:
+        Standard:
+          create:
+            implementation: mysql/mysql_dbms_install.sh
+            inputs:
+              db_root_password: passw0rd
+          start: mysql/mysql_dbms_start.sh
+          configure:
+            implementation: mysql/mysql_dbms_configure.sh
+            inputs:
+              db_port: 3366
+
+    webserver:
+      type: tosca.nodes.WebServer
+      requirements:
+        - host: server
+      interfaces:
+        Standard:
+          create: webserver/webserver_install.sh
+          start: webserver/webserver_start.sh
+    server:
+      type: tosca.nodes.Compute
+      capabilities:
+        host:
+          properties:
+            disk_size: 10 GB
+            num_cpus: { get_input: cpus }
+            mem_size: 4096 MB
+        os:
+          properties:
+            architecture: x86_64
+            type: Linux
+            distribution: Ubuntu
+            version: 14.04
+
+  outputs:
+    website_url:
+      description: URL for Wordpress wiki.
+      value: { get_attribute: [server, private_address] }

translator/tests/test_tosca_hot_translation.py

@@ -323,3 +323,80 @@ class ToscaHotTranslationTest(TestCase):
                                                                    params)
         self.assertEqual({}, diff, '<difference> : ' +
                          json.dumps(diff, indent=4, separators=(', ', ': ')))
+
+    def test_hot_translate_template_with_url_import(self):
+        tosca_file = '../tests/data/' \
+                     'tosca_single_instance_wordpress_with_url_import.yaml'
+        hot_file = '../tests/data/hot_output/' \
+                   'hot_single_instance_wordpress.yaml'
+        params = {'db_name': 'wordpress',
+                  'db_user': 'wp_user',
+                  'db_pwd': 'wp_pass',
+                  'db_root_pwd': 'passw0rd',
+                  'db_port': 3366,
+                  'cpus': 8}
+        diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
+                                                                   hot_file,
+                                                                   params)
+        self.assertEqual({}, diff, '<difference> : ' +
+                         json.dumps(diff, indent=4, separators=(', ', ': ')))
+
+    def test_hot_translate_template_by_url_with_local_import(self):
+        tosca_file = 'https://raw.githubusercontent.com/openstack/' \
+                     'heat-translator/master/translator/tests/data/' \
+                     'tosca_single_instance_wordpress.yaml'
+        hot_file = '../tests/data/hot_output/' \
+                   'hot_single_instance_wordpress.yaml'
+        params = {'db_name': 'wordpress',
+                  'db_user': 'wp_user',
+                  'db_pwd': 'wp_pass',
+                  'db_root_pwd': 'passw0rd',
+                  'db_port': 3366,
+                  'cpus': 8}
+        diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file,
+                                                                   hot_file,
+                                                                   params)
+        self.assertEqual({}, diff, '<difference> : ' +
+                         json.dumps(diff, indent=4, separators=(', ', ': ')))
+
+    def test_hot_translate_template_by_url_with_local_abspath_import(self):
+        tosca_file = 'https://ibm.box.com/shared/static/' \
+                     'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml'
+        hot_file = '../tests/data/hot_output/' \
+                   'hot_single_instance_wordpress.yaml'
+        params = {'db_name': 'wordpress',
+                  'db_user': 'wp_user',
+                  'db_pwd': 'wp_pass',
+                  'db_root_pwd': 'passw0rd',
+                  'db_port': 3366,
+                  'cpus': 8}
+        try:
+            TranslationUtils.compare_tosca_translation_with_hot(
+                tosca_file, hot_file, params)
+        except Exception as err:
+            self.assertTrue(isinstance(err, ImportError))
+            self.assertEqual(
+                'Absolute file name /tmp/wordpress.yaml cannot be used for a '
+                'URL-based input https://ibm.box.com/shared/static/'
+                'lrgdktp9vw3991y2hlogmghwwvnok3lu.yaml template.',
+                err.__str__())
+        else:
+            raise Exception(
+                'The unit test that was expected to fail did not fail.')
+
+    def test_hot_translate_template_by_url_with_url_import(self):
+        tosca_url = 'https://ibm.box.com/shared/static/' \
+                    'tocmxe9b9x7to0lj5ph9mx58d47ol77m.yaml'
+        hot_file = '../tests/data/hot_output/' \
+                   'hot_single_instance_wordpress.yaml'
+        params = {'db_name': 'wordpress',
+                  'db_user': 'wp_user',
+                  'db_pwd': 'wp_pass',
+                  'db_root_pwd': 'passw0rd',
+                  'db_port': 3366,
+                  'cpus': 8}
+        diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_url,
+                                                                   hot_file,
+                                                                   params)
+        self.assertEqual({}, diff, '<difference> : ' +
+                         json.dumps(diff, indent=4, separators=(', ', ': ')))

translator/tests/test_utils.py

@@ -20,6 +20,7 @@ class CommonUtilsTest(TestCase):
     MemoryUnit = translator.common.utils.MemoryUnit
     cmpUtils = translator.common.utils.CompareUtils
     yamlUtils = translator.common.utils.YamlUtils
+    UrlUtils = translator.common.utils.UrlUtils
 
     def test_convert_unit_size_to_num(self):
         size = '1 TB'
@@ -227,3 +228,12 @@ class CommonUtilsTest(TestCase):
         value = 1
         output = translator.common.utils.str_to_num(value)
         self.assertEqual(value, output)
+
+    def test_urlutils_validate_url(self):
+        self.assertTrue(self.UrlUtils.validate_url("http://www.github.com/"))
+        self.assertTrue(
+            self.UrlUtils.validate_url("https://github.com:81/a/2/a.b"))
+        self.assertTrue(self.UrlUtils.validate_url("ftp://github.com"))
+        self.assertFalse(self.UrlUtils.validate_url("github.com"))
+        self.assertFalse(self.UrlUtils.validate_url("123"))
+        self.assertFalse(self.UrlUtils.validate_url("a/b/c"))