Add a make_url intrinsic function

A large proportion of uses of the str_replace function is to build URLs out
of various components from various sources. This is invariably brittle,
with a failure to escape special characters, deal with IPv6 addresses, and
so on. The make_url function provides a both a tidier interface and correct
handling of these edge cases.

Change-Id: I61b6dff01cd509b3d1c54bca118632c276569f4e

doc/source/template_guide/hot_spec.rst

@@ -293,7 +293,8 @@ for the ``heat_template_version`` key:
 -------------------
     The key with value ``2017-09-01`` or ``pike`` indicates that the YAML
     document is a HOT template and it may contain features added and/or removed
-    up until the Pike release. The complete list of supported functions is::
+    up until the Pike release. This version adds the ``make_url`` function for
+    assembling URLs. The complete list of supported functions is::
 
       digest
       filter
@@ -302,6 +303,7 @@ for the ``heat_template_version`` key:
       get_param
       get_resource
       list_join
+      make_url
       map_merge
       map_replace
       repeat
@@ -1845,3 +1847,47 @@ For example
             - {get_param: list_param}
 
 output_list will be evaluated to [1, 2].
+
+make_url
+--------
+
+The ``make_url`` function builds URLs.
+
+The syntax of the ``make_url`` function is
+
+.. code-block:: yaml
+
+    make_url:
+      scheme: <protocol>
+      username: <username>
+      password: <password>
+      host: <hostname or IP>
+      port: <port>
+      path: <path>
+      query:
+        <key1>: <value1>
+        <key2>: <value2>
+      fragment: <fragment>
+
+
+All parameters are optional.
+
+For example
+
+.. code-block:: yaml
+
+    outputs:
+      server_url:
+        value:
+          make_url:
+            scheme: http
+            host: {get_attr: [server, networks, <network_name>, 0]}
+            port: 8080
+            path: /hello
+            query:
+              recipient: world
+            fragment: greeting
+
+``server_url`` will be evaluated to a URL in the form::
+
+    http://[<server IP>]:8080/hello?recipient=world#greeting

heat/engine/hot/functions.py

@@ -18,6 +18,7 @@ import itertools
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 import six
+from six.moves.urllib import parse as urlparse
 import yaql
 from yaql.language import exceptions
 
@@ -1320,3 +1321,119 @@ class Filter(function.Function):
             raise TypeError(
                 _('"%(fn)s" filters a list of values') % self.fn_name)
         return [i for i in sequence if i not in values]
+
+
+class MakeURL(function.Function):
+    """A function for performing substitutions on maps.
+
+    Takes the form::
+
+        make_url:
+          scheme: <protocol>
+          username: <username>
+          password: <password>
+          host: <hostname or IP>
+          port: <port>
+          path: <path>
+          query:
+            <key1>: <value1>
+          fragment: <fragment>
+
+    And resolves to a correctly-escaped URL constructed from the various
+    components.
+    """
+
+    _ARG_KEYS = (
+        SCHEME, USERNAME, PASSWORD, HOST, PORT,
+        PATH, QUERY, FRAGMENT,
+    ) = (
+        'scheme', 'username', 'password', 'host', 'port',
+        'path', 'query', 'fragment',
+    )
+
+    def _check_args(self, args):
+        for arg in self._ARG_KEYS:
+            if arg in args:
+                if arg == self.QUERY:
+                    if not isinstance(args[arg], (function.Function,
+                                                  collections.Mapping)):
+                        raise TypeError(_('The "%(arg)s" argument to '
+                                          '"(fn_name)%s" must be a map') %
+                                        {'arg': arg,
+                                         'fn_name': self.fn_name})
+                    return
+                elif arg == self.PORT:
+                    port = args[arg]
+                    if not isinstance(port, function.Function):
+                        if not isinstance(port, six.integer_types):
+                            try:
+                                port = int(port)
+                            except ValueError:
+                                raise ValueError(_('Invalid URL port "%s"') %
+                                                 port)
+                        if not (0 < port <= 65535):
+                            raise ValueError(_('Invalid URL port %d') % port)
+                else:
+                    if not isinstance(args[arg], (function.Function,
+                                                  six.string_types)):
+                        raise TypeError(_('The "%(arg)s" argument to '
+                                          '"(fn_name)%s" must be a string') %
+                                        {'arg': arg,
+                                         'fn_name': self.fn_name})
+
+    def validate(self):
+        super(MakeURL, self).validate()
+
+        if not isinstance(self.args, collections.Mapping):
+            raise TypeError(_('The arguments to "%s" must '
+                              'be a map') % self.fn_name)
+
+        invalid_keys = set(self.args) - set(self._ARG_KEYS)
+        if invalid_keys:
+            raise ValueError(_('Invalid arguments to "%(fn)s": %(args)s') %
+                             {'fn': self.fn_name,
+                              'args': ', '.join(invalid_keys)})
+
+        self._check_args(self.args)
+
+    def result(self):
+        args = function.resolve(self.args)
+        self._check_args(args)
+
+        scheme = args.get(self.SCHEME, '')
+        if ':' in scheme:
+            raise ValueError(_('URL "%s" should not contain \':\'') %
+                             self.SCHEME)
+
+        def netloc():
+            username = urlparse.quote(args.get(self.USERNAME, ''), safe='')
+            password = urlparse.quote(args.get(self.PASSWORD, ''), safe='')
+            if username or password:
+                yield username
+                if password:
+                    yield ':'
+                    yield password
+                yield '@'
+
+            host = args.get(self.HOST, '')
+            if host.startswith('[') and host.endswith(']'):
+                host = host[1:-1]
+            host = urlparse.quote(host, safe=':')
+            if ':' in host:
+                host = '[%s]' % host
+            yield host
+
+            port = args.get(self.PORT, '')
+            if port:
+                yield ':'
+                yield six.text_type(port)
+
+        path = urlparse.quote(args.get(self.PATH, ''))
+
+        query_dict = args.get(self.QUERY, {})
+        query = urlparse.urlencode(query_dict)
+
+        fragment = urlparse.quote(args.get(self.FRAGMENT, ''))
+
+        return urlparse.urlunsplit((scheme, ''.join(netloc()),
+                                    path, query, fragment))

heat/engine/hot/template.py

@@ -588,6 +588,9 @@ class HOTemplate20170901(HOTemplate20170224):
         'filter': hot_funcs.Filter,
         'str_replace_strict': hot_funcs.ReplaceJsonStrict,
 
+        # functions added in 2017-09-01
+        'make_url': hot_funcs.MakeURL,
+
         # functions removed from 2015-10-15
         'Fn::Select': hot_funcs.Removed,
 

heat/tests/test_hot.py

@@ -1810,6 +1810,231 @@ conditions:
         stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl)
         self.assertRaises(TypeError, self.resolve, snippet, tmpl, stack=stack)
 
+    def test_make_url_basic(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': 'example.com',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        func = tmpl.parse(None, snippet)
+        function.validate(func)
+        resolved = function.resolve(func)
+
+        self.assertEqual('http://example.com/foo/bar',
+                         resolved)
+
+    def test_make_url_ipv6(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': '::1',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('http://[::1]/foo/bar',
+                         resolved)
+
+    def test_make_url_ipv6_ready(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': '[::1]',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('http://[::1]/foo/bar',
+                         resolved)
+
+    def test_make_url_port_string(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'https',
+                'host': 'example.com',
+                'port': '80',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('https://example.com:80/foo/bar',
+                         resolved)
+
+    def test_make_url_port_int(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'https',
+                'host': 'example.com',
+                'port': 80,
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('https://example.com:80/foo/bar',
+                         resolved)
+
+    def test_make_url_port_invalid_high(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'https',
+                'host': 'example.com',
+                'port': 100000,
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        self.assertRaises(ValueError, self.resolve, snippet, tmpl)
+
+    def test_make_url_port_invalid_low(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'https',
+                'host': 'example.com',
+                'port': '0',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        self.assertRaises(ValueError, self.resolve, snippet, tmpl)
+
+    def test_make_url_port_invalid_string(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'https',
+                'host': 'example.com',
+                'port': '1.1',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        self.assertRaises(ValueError, self.resolve, snippet, tmpl)
+
+    def test_make_url_username(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'username': 'wibble',
+                'host': 'example.com',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('http://wibble@example.com/foo/bar',
+                         resolved)
+
+    def test_make_url_username_password(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'username': 'wibble',
+                'password': 'blarg',
+                'host': 'example.com',
+                'path': '/foo/bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('http://wibble:blarg@example.com/foo/bar',
+                         resolved)
+
+    def test_make_url_query(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': 'example.com',
+                'path': '/foo/?bar',
+                'query': {
+                    'foo': 'bar&baz',
+                    'blarg': 'wib=ble',
+                },
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertIn(resolved,
+                      ['http://example.com/foo/%3Fbar'
+                       '?foo=bar%26baz&blarg=wib%3Dble',
+                       'http://example.com/foo/%3Fbar'
+                       '?blarg=wib%3Dble&foo=bar%26baz'])
+
+    def test_make_url_fragment(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': 'example.com',
+                'path': 'foo/bar',
+                'fragment': 'baz'
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('http://example.com/foo/bar#baz',
+                         resolved)
+
+    def test_make_url_file(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'file',
+                'path': 'foo/bar'
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('file:///foo/bar',
+                         resolved)
+
+    def test_make_url_file_leading_slash(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'file',
+                'path': '/foo/bar'
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        resolved = self.resolve(snippet, tmpl)
+
+        self.assertEqual('file:///foo/bar',
+                         resolved)
+
+    def test_make_url_bad_args_type(self):
+        snippet = {
+            'make_url': 'http://example.com/foo/bar'
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        func = tmpl.parse(None, snippet)
+        self.assertRaises(exception.StackValidationFailed, function.validate,
+                          func)
+
+    def test_make_url_invalid_key(self):
+        snippet = {
+            'make_url': {
+                'scheme': 'http',
+                'host': 'example.com',
+                'foo': 'bar',
+            }
+        }
+        tmpl = template.Template(hot_pike_tpl_empty)
+        func = tmpl.parse(None, snippet)
+        self.assertRaises(exception.StackValidationFailed, function.validate,
+                          func)
+
     def test_depends_condition(self):
         hot_tpl = template_format.parse('''
         heat_template_version: 2016-10-14

releasenotes/notes/make_url-function-d76737adb1e54801.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - The Pike version of HOT (2017-09-01) adds a make_url function to simplify
+    combining data from different sources into a URL with correct handling for
+    escaping and IPv6 addresses.