Browse Source

Fix get_file in out-of-tree templates

We already have special processing in place for out-of-tree environment
files and templates, but it didn't handle `get_file` (or `type`)
links. This commit adds that handling.

Heatclient automatically uses absolute `file:///` links when processing
the external environment files and other files referenced from them via
`get_file` or `type`. Because we upload all our environment files and
templates to Swift, such links don't work. We need to use relative links
in `get_file` and `type`.

Change-Id: I009f75cbc6278a0a2ff75e93e1ed44f2c4893783
Closes-Bug: #1631426
Jiri Stransky 3 years ago
3 changed files with 144 additions and 9 deletions
  1. +67
  2. +66
  3. +11

+ 67
- 0
tripleoclient/tests/ View File

@@ -19,6 +19,7 @@ import mock
import os.path
import tempfile
from unittest import TestCase
import yaml

from tripleoclient import exceptions
from tripleoclient.tests.v1.utils import (
@@ -718,3 +719,69 @@ class TestAssignVerifyProfiles(TestCase):
self.nodes[:] = [self._get_fake_node(profile=None)]
self.flavors = {'baremetal': (FakeFlavor('baremetal', None), 1)}
self._test(0, 0)

class TestReplaceLinks(TestCase):

def setUp(self):
super(TestReplaceLinks, self).setUp()
self.link_replacement = {

def test_replace_links(self):
source = (
'description: my template\n'
'heat_template_version: "2014-10-16"\n'
' test_config:\n'
' properties:\n'
' config: {get_file: "file:///home/stack/"}\n'
' type: OS::Heat::SoftwareConfig\n'
expected = (
'description: my template\n'
'heat_template_version: "2014-10-16"\n'
' test_config:\n'
' properties:\n'
' config: {get_file: user-files/home/stack/}\n'
' type: OS::Heat::SoftwareConfig\n'

# the yaml->string dumps aren't always character-precise, so
# we need to parse them into dicts for comparison
expected_dict = yaml.safe_load(expected)
result_dict = yaml.safe_load(utils.replace_links_in_template_contents(
source, self.link_replacement))
self.assertEqual(expected_dict, result_dict)

def test_replace_links_not_template(self):
# valid JSON/YAML, but doesn't have heat_template_version
source = '{"get_file": "file:///home/stack/"}'
source, self.link_replacement))

def test_replace_links_not_yaml(self):
# invalid JSON/YAML -- curly brace left open
source = '{"invalid JSON"'
source, self.link_replacement))

def test_relative_link_replacement(self):
current_dir = 'user-files/home/stack'
expected = {
self.assertEqual(expected, utils.relative_link_replacement(
self.link_replacement, current_dir))

+ 66
- 0
tripleoclient/ View File

@@ -900,3 +900,69 @@ def parse_env_file(env_file, file_type=None):
nodes_config = nodes_config['nodes']

return nodes_config

def replace_links_in_template_contents(contents, link_replacement):
"""Replace get_file and type file links in Heat template contents

If the string contents passed in is a Heat template, scan the
template for 'get_file' and 'type' occurences, and replace the
file paths according to link_replacement dict. (Key/value in
link_replacement are from/to, respectively.)

If the string contents don't look like a Heat template, return the
contents unmodified.

template = {}
template = yaml.safe_load(contents)
except yaml.YAMLError:
return contents

if not (isinstance(template, dict) and
return contents

template = replace_links_in_template(template, link_replacement)

return yaml.safe_dump(template)

def replace_links_in_template(template_part, link_replacement):
"""Replace get_file and type file links in a Heat template

Scan the template for 'get_file' and 'type' occurences, and
replace the file paths according to link_replacement
dict. (Key/value in link_replacement are from/to, respectively.)

def replaced_dict_value(key, value):
if ((key == 'get_file' or key == 'type') and
isinstance(value, six.string_types)):
return link_replacement.get(value, value)
return replace_links_in_template(value, link_replacement)

def replaced_list_value(value):
return replace_links_in_template(value, link_replacement)

if isinstance(template_part, dict):
return {k: replaced_dict_value(k, v)
for k, v in six.iteritems(template_part)}
elif isinstance(template_part, list):
return map(replaced_list_value, template_part)
return template_part

def relative_link_replacement(link_replacement, current_dir):
"""Generate a relative version of link_replacement dictionary.

Get a link_replacement dictionary (where key/value are from/to
respectively), and make the values in that dictionary relative
paths with respect to current_dir.

return {k: os.path.relpath(v, current_dir)
for k, v in six.iteritems(link_replacement)}

+ 11
- 9
tripleoclient/v1/ View File

@@ -16,7 +16,6 @@ from __future__ import print_function

import argparse
import glob
import hashlib
import logging
import os
import os.path
@@ -360,7 +359,8 @@ class DeployOvercloud(command.Command):
file_relocation = {}
file_prefix = "file://"

for fullpath, contents in files_dict.items():
# select files files for relocation & upload
for fullpath in files_dict.keys():

if not fullpath.startswith(file_prefix):
@@ -371,13 +371,15 @@ class DeployOvercloud(command.Command):
# This should already be uploaded.

filename = os.path.basename(path)
checksum = hashlib.md5()
digest = checksum.hexdigest()
swift_path = "user-files/{}-{}".format(digest, filename)
swift_client.put_object(container_name, swift_path, contents)
file_relocation[fullpath] = swift_path
file_relocation[fullpath] = "user-files/{}".format(path[1:])

# make sure links within files point to new locations, and upload them
for orig_path, reloc_path in file_relocation.items():
link_replacement = utils.relative_link_replacement(
file_relocation, os.path.dirname(reloc_path))
contents = utils.replace_links_in_template_contents(
files_dict[orig_path], link_replacement)
swift_client.put_object(container_name, reloc_path, contents)

return file_relocation