Fix get_file in out-of-tree templates

We already have special processing in place for out-of-tree environment
files and templates, but it didn't handle `get_file` (or `type`)
links. This commit adds that handling.

Heatclient automatically uses absolute `file:///` links when processing
the external environment files and other files referenced from them via
`get_file` or `type`. Because we upload all our environment files and
templates to Swift, such links don't work. We need to use relative links
in `get_file` and `type`.

Change-Id: I009f75cbc6278a0a2ff75e93e1ed44f2c4893783
Closes-Bug: #1631426

tripleoclient/tests/test_utils.py

@@ -19,6 +19,7 @@ import mock
 import os.path
 import tempfile
 from unittest import TestCase
+import yaml
 
 from tripleoclient import exceptions
 from tripleoclient.tests.v1.utils import (
@@ -718,3 +719,69 @@ class TestAssignVerifyProfiles(TestCase):
         self.nodes[:] = [self._get_fake_node(profile=None)]
         self.flavors = {'baremetal': (FakeFlavor('baremetal', None), 1)}
         self._test(0, 0)
+
+
+class TestReplaceLinks(TestCase):
+
+    def setUp(self):
+        super(TestReplaceLinks, self).setUp()
+        self.link_replacement = {
+            'file:///home/stack/test.sh':
+                'user-files/home/stack/test.sh',
+            'file:///usr/share/extra-templates/my.yml':
+                'user-files/usr/share/extra-templates/my.yml',
+        }
+
+    def test_replace_links(self):
+        source = (
+            'description: my template\n'
+            'heat_template_version: "2014-10-16"\n'
+            'resources:\n'
+            '  test_config:\n'
+            '    properties:\n'
+            '      config: {get_file: "file:///home/stack/test.sh"}\n'
+            '    type: OS::Heat::SoftwareConfig\n'
+        )
+        expected = (
+            'description: my template\n'
+            'heat_template_version: "2014-10-16"\n'
+            'resources:\n'
+            '  test_config:\n'
+            '    properties:\n'
+            '      config: {get_file: user-files/home/stack/test.sh}\n'
+            '    type: OS::Heat::SoftwareConfig\n'
+        )
+
+        # the yaml->string dumps aren't always character-precise, so
+        # we need to parse them into dicts for comparison
+        expected_dict = yaml.safe_load(expected)
+        result_dict = yaml.safe_load(utils.replace_links_in_template_contents(
+            source, self.link_replacement))
+        self.assertEqual(expected_dict, result_dict)
+
+    def test_replace_links_not_template(self):
+        # valid JSON/YAML, but doesn't have heat_template_version
+        source = '{"get_file": "file:///home/stack/test.sh"}'
+        self.assertEqual(
+            source,
+            utils.replace_links_in_template_contents(
+                source, self.link_replacement))
+
+    def test_replace_links_not_yaml(self):
+        # invalid JSON/YAML -- curly brace left open
+        source = '{"invalid JSON"'
+        self.assertEqual(
+            source,
+            utils.replace_links_in_template_contents(
+                source, self.link_replacement))
+
+    def test_relative_link_replacement(self):
+        current_dir = 'user-files/home/stack'
+        expected = {
+            'file:///home/stack/test.sh':
+                'test.sh',
+            'file:///usr/share/extra-templates/my.yml':
+                '../../usr/share/extra-templates/my.yml',
+        }
+        self.assertEqual(expected, utils.relative_link_replacement(
+            self.link_replacement, current_dir))

tripleoclient/utils.py

@@ -900,3 +900,69 @@ def parse_env_file(env_file, file_type=None):
         nodes_config = nodes_config['nodes']
 
     return nodes_config
+
+
+def replace_links_in_template_contents(contents, link_replacement):
+    """Replace get_file and type file links in Heat template contents
+
+    If the string contents passed in is a Heat template, scan the
+    template for 'get_file' and 'type' occurences, and replace the
+    file paths according to link_replacement dict. (Key/value in
+    link_replacement are from/to, respectively.)
+
+    If the string contents don't look like a Heat template, return the
+    contents unmodified.
+    """
+
+    template = {}
+    try:
+        template = yaml.safe_load(contents)
+    except yaml.YAMLError:
+        return contents
+
+    if not (isinstance(template, dict) and
+            template.get('heat_template_version')):
+        return contents
+
+    template = replace_links_in_template(template, link_replacement)
+
+    return yaml.safe_dump(template)
+
+
+def replace_links_in_template(template_part, link_replacement):
+    """Replace get_file and type file links in a Heat template
+
+    Scan the template for 'get_file' and 'type' occurences, and
+    replace the file paths according to link_replacement
+    dict. (Key/value in link_replacement are from/to, respectively.)
+    """
+
+    def replaced_dict_value(key, value):
+        if ((key == 'get_file' or key == 'type') and
+                isinstance(value, six.string_types)):
+            return link_replacement.get(value, value)
+        else:
+            return replace_links_in_template(value, link_replacement)
+
+    def replaced_list_value(value):
+        return replace_links_in_template(value, link_replacement)
+
+    if isinstance(template_part, dict):
+        return {k: replaced_dict_value(k, v)
+                for k, v in six.iteritems(template_part)}
+    elif isinstance(template_part, list):
+        return map(replaced_list_value, template_part)
+    else:
+        return template_part
+
+
+def relative_link_replacement(link_replacement, current_dir):
+    """Generate a relative version of link_replacement dictionary.
+
+    Get a link_replacement dictionary (where key/value are from/to
+    respectively), and make the values in that dictionary relative
+    paths with respect to current_dir.
+    """
+
+    return {k: os.path.relpath(v, current_dir)
+            for k, v in six.iteritems(link_replacement)}

tripleoclient/v1/overcloud_deploy.py

@@ -16,7 +16,6 @@ from __future__ import print_function
 
 import argparse
 import glob
-import hashlib
 import logging
 import os
 import os.path
@@ -360,7 +359,8 @@ class DeployOvercloud(command.Command):
         file_relocation = {}
         file_prefix = "file://"
 
-        for fullpath, contents in files_dict.items():
+        # select files files for relocation & upload
+        for fullpath in files_dict.keys():
 
             if not fullpath.startswith(file_prefix):
                 continue
@@ -371,13 +371,15 @@ class DeployOvercloud(command.Command):
                 # This should already be uploaded.
                 continue
 
-            filename = os.path.basename(path)
+            file_relocation[fullpath] = "user-files/{}".format(path[1:])
-            checksum = hashlib.md5()
+
-            checksum.update(path)
+        # make sure links within files point to new locations, and upload them
-            digest = checksum.hexdigest()
+        for orig_path, reloc_path in file_relocation.items():
-            swift_path = "user-files/{}-{}".format(digest, filename)
+            link_replacement = utils.relative_link_replacement(
-            swift_client.put_object(container_name, swift_path, contents)
+                file_relocation, os.path.dirname(reloc_path))
-            file_relocation[fullpath] = swift_path
+            contents = utils.replace_links_in_template_contents(
+                files_dict[orig_path], link_replacement)
+            swift_client.put_object(container_name, reloc_path, contents)
 
         return file_relocation