A Python library for code common to TripleO CLI and TripleO UI.
Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

402 Zeilen
17KB

  1. # Copyright 2016 Red Hat, Inc.
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. import jinja2
  16. import json
  17. import logging
  18. import os
  19. import requests
  20. import six
  21. import tempfile as tf
  22. import yaml
  23. from heatclient.common import template_utils
  24. from mistral_lib import actions
  25. from swiftclient import exceptions as swiftexceptions
  26. from tripleo_common.actions import base
  27. from tripleo_common import constants
  28. from tripleo_common.utils import plan as plan_utils
  29. from tripleo_common.utils import tarball
  30. LOG = logging.getLogger(__name__)
  31. def _create_temp_file(data):
  32. handle, env_temp_file = tf.mkstemp()
  33. with open(env_temp_file, 'w') as temp_file:
  34. temp_file.write(json.dumps(data))
  35. os.close(handle)
  36. return env_temp_file
  37. class J2SwiftLoader(jinja2.BaseLoader):
  38. """Jinja2 loader to fetch included files from swift
  39. This attempts to fetch a template include file from the given container.
  40. An optional search path or list of search paths can be provided. By default
  41. only the absolute path relative to the container root is searched.
  42. """
  43. def __init__(self, swift, container, searchpath=None):
  44. self.swift = swift
  45. self.container = container
  46. if searchpath is not None:
  47. if isinstance(searchpath, six.string_types):
  48. self.searchpath = [searchpath]
  49. else:
  50. self.searchpath = list(searchpath)
  51. else:
  52. self.searchpath = []
  53. # Always search the absolute path from the root of the swift container
  54. if '' not in self.searchpath:
  55. self.searchpath.append('')
  56. def get_source(self, environment, template):
  57. pieces = jinja2.loaders.split_template_path(template)
  58. for searchpath in self.searchpath:
  59. template_path = os.path.join(searchpath, *pieces)
  60. try:
  61. source = self.swift.get_object(
  62. self.container, template_path)[1]
  63. return source, None, False
  64. except swiftexceptions.ClientException:
  65. pass
  66. raise jinja2.exceptions.TemplateNotFound(template)
  67. class UploadTemplatesAction(base.TripleOAction):
  68. """Upload default heat templates for TripleO."""
  69. def __init__(self, container=constants.DEFAULT_CONTAINER_NAME,
  70. templates_path=constants.DEFAULT_TEMPLATES_PATH):
  71. super(UploadTemplatesAction, self).__init__()
  72. self.container = container
  73. self.templates_path = templates_path
  74. def run(self, context):
  75. with tf.NamedTemporaryFile() as tmp_tarball:
  76. tarball.create_tarball(self.templates_path, tmp_tarball.name)
  77. tarball.tarball_extract_to_swift_container(
  78. self.get_object_client(context),
  79. tmp_tarball.name,
  80. self.container)
  81. class ProcessTemplatesAction(base.TripleOAction):
  82. """Process Templates and Environments
  83. This method processes the templates and files in a given deployment
  84. plan into a format that can be passed to Heat.
  85. """
  86. def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
  87. super(ProcessTemplatesAction, self).__init__()
  88. self.container = container
  89. def _j2_render_and_put(self,
  90. j2_template,
  91. j2_data,
  92. outfile_name=None,
  93. context=None):
  94. swift = self.get_object_client(context)
  95. yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
  96. # Search for templates relative to the current template path first
  97. template_base = os.path.dirname(yaml_f)
  98. j2_loader = J2SwiftLoader(swift, self.container, template_base)
  99. try:
  100. # Render the j2 template
  101. template = jinja2.Environment(loader=j2_loader).from_string(
  102. j2_template)
  103. r_template = template.render(**j2_data)
  104. except jinja2.exceptions.TemplateError as ex:
  105. error_msg = ("Error rendering template %s : %s"
  106. % (yaml_f, six.text_type(ex)))
  107. LOG.error(error_msg)
  108. raise Exception(error_msg)
  109. try:
  110. # write the template back to the plan container
  111. LOG.info("Writing rendered template %s" % yaml_f)
  112. self.cache_delete(context,
  113. self.container,
  114. "tripleo.parameters.get")
  115. swift.put_object(
  116. self.container, yaml_f, r_template)
  117. except swiftexceptions.ClientException as ex:
  118. error_msg = ("Error storing file %s in container %s"
  119. % (yaml_f, self.container))
  120. LOG.error(error_msg)
  121. raise Exception(error_msg)
  122. def _get_j2_excludes_file(self, context):
  123. swift = self.get_object_client(context)
  124. try:
  125. j2_excl_file = swift.get_object(
  126. self.container, constants.OVERCLOUD_J2_EXCLUDES)[1]
  127. j2_excl_data = yaml.safe_load(j2_excl_file)
  128. if (j2_excl_data is None or j2_excl_data.get('name') is None):
  129. j2_excl_data = {"name": []}
  130. LOG.info("j2_excludes.yaml is either empty or there are "
  131. "no templates to exclude, defaulting the J2 "
  132. "excludes list to: %s" % j2_excl_data)
  133. except swiftexceptions.ClientException:
  134. j2_excl_data = {"name": []}
  135. LOG.info("No J2 exclude file found, defaulting "
  136. "the J2 excludes list to: %s" % j2_excl_data)
  137. return j2_excl_data
  138. def _process_custom_roles(self, context):
  139. swift = self.get_object_client(context)
  140. try:
  141. j2_role_file = swift.get_object(
  142. self.container, constants.OVERCLOUD_J2_ROLES_NAME)[1]
  143. role_data = yaml.safe_load(j2_role_file)
  144. except swiftexceptions.ClientException:
  145. LOG.info("No %s file found, skipping jinja templating"
  146. % constants.OVERCLOUD_J2_ROLES_NAME)
  147. return
  148. try:
  149. j2_network_file = swift.get_object(
  150. self.container, constants.OVERCLOUD_J2_NETWORKS_NAME)[1]
  151. network_data = yaml.safe_load(j2_network_file)
  152. except swiftexceptions.ClientException:
  153. # Until t-h-t contains network_data.yaml we tolerate a missing file
  154. LOG.warning("No %s file found, ignoring"
  155. % constants.OVERCLOUD_J2_ROLES_NAME)
  156. network_data = []
  157. j2_excl_data = self._get_j2_excludes_file(context)
  158. try:
  159. # Iterate over all files in the plan container
  160. # we j2 render any with the .j2.yaml suffix
  161. container_files = swift.get_container(self.container)
  162. except swiftexceptions.ClientException as ex:
  163. error_msg = ("Error listing contents of container %s : %s"
  164. % (self.container, six.text_type(ex)))
  165. LOG.error(error_msg)
  166. raise Exception(error_msg)
  167. role_names = [r.get('name') for r in role_data]
  168. r_map = {}
  169. for r in role_data:
  170. r_map[r.get('name')] = r
  171. excl_templates = j2_excl_data.get('name')
  172. n_map = {}
  173. for n in network_data:
  174. if (n.get('enabled') is not False):
  175. n_map[n.get('name')] = n
  176. if not n.get('name_lower'):
  177. n_map[n.get('name')]['name_lower'] = n.get('name').lower()
  178. else:
  179. LOG.info("skipping %s network: network is disabled." %
  180. n.get('name'))
  181. for f in [f.get('name') for f in container_files[1]]:
  182. # We do three templating passes here:
  183. # 1. *.role.j2.yaml - we template just the role name
  184. # and create multiple files (one per role)
  185. # 2 *.network.j2.yaml - we template the network name and
  186. # data and create multiple files for networks and
  187. # network ports (one per network)
  188. # 3. *.j2.yaml - we template with all roles_data,
  189. # and create one file common to all roles
  190. if f.endswith('.role.j2.yaml'):
  191. LOG.info("jinja2 rendering role template %s" % f)
  192. j2_template = swift.get_object(self.container, f)[1]
  193. LOG.info("jinja2 rendering roles %s" % ","
  194. .join(role_names))
  195. for role in role_names:
  196. LOG.info("jinja2 rendering role %s" % role)
  197. out_f = "-".join(
  198. [role.lower(),
  199. os.path.basename(f).replace('.role.j2.yaml',
  200. '.yaml')])
  201. out_f_path = os.path.join(os.path.dirname(f), out_f)
  202. if not (out_f_path in excl_templates):
  203. if '{{role.name}}' in j2_template:
  204. j2_data = {'role': r_map[role],
  205. 'networks': network_data}
  206. self._j2_render_and_put(j2_template,
  207. j2_data,
  208. out_f_path,
  209. context=context)
  210. else:
  211. # Backwards compatibility with templates
  212. # that specify {{role}} vs {{role.name}}
  213. j2_data = {'role': role, 'networks': network_data}
  214. LOG.debug("role legacy path for role %s" % role)
  215. if r_map[role].get('disable_constraints', False):
  216. j2_data['disable_constraints'] = True
  217. self._j2_render_and_put(j2_template,
  218. j2_data,
  219. out_f_path,
  220. context=context)
  221. else:
  222. LOG.info("Skipping rendering of %s, defined in %s" %
  223. (out_f_path, j2_excl_data))
  224. elif (f.endswith('.network.j2.yaml')):
  225. LOG.info("jinja2 rendering network template %s" % f)
  226. j2_template = swift.get_object(self.container, f)[1]
  227. LOG.info("jinja2 rendering networks %s" % ",".join(n_map))
  228. for network in n_map:
  229. j2_data = {'network': n_map[network]}
  230. # Output file names in "<name>.yaml" format
  231. out_f = os.path.basename(f).replace('.network.j2.yaml',
  232. '.yaml')
  233. if os.path.dirname(f).endswith('ports'):
  234. out_f = out_f.replace('port',
  235. n_map[network]['name_lower'])
  236. else:
  237. out_f = out_f.replace('network',
  238. n_map[network]['name_lower'])
  239. out_f_path = os.path.join(os.path.dirname(f), out_f)
  240. if not (out_f_path in excl_templates):
  241. self._j2_render_and_put(j2_template,
  242. j2_data,
  243. out_f_path,
  244. context=context)
  245. else:
  246. LOG.info("Skipping rendering of %s, defined in %s" %
  247. (out_f_path, j2_excl_data))
  248. elif f.endswith('.j2.yaml'):
  249. LOG.info("jinja2 rendering %s" % f)
  250. j2_template = swift.get_object(self.container, f)[1]
  251. j2_data = {'roles': role_data, 'networks': network_data}
  252. out_f = f.replace('.j2.yaml', '.yaml')
  253. self._j2_render_and_put(j2_template,
  254. j2_data,
  255. out_f,
  256. context=context)
  257. def run(self, context):
  258. error_text = None
  259. self.context = context
  260. swift = self.get_object_client(context)
  261. try:
  262. plan_env = plan_utils.get_env(swift, self.container)
  263. except swiftexceptions.ClientException as err:
  264. err_msg = ("Error retrieving environment for plan %s: %s" % (
  265. self.container, err))
  266. LOG.exception(err_msg)
  267. return actions.Result(error=error_text)
  268. try:
  269. # if the jinja overcloud template exists, process it and write it
  270. # back to the swift container before continuing processing. The
  271. # method called below should handle the case where the files are
  272. # not found in swift, but if they are found and an exception
  273. # occurs during processing, that exception will cause the
  274. # ProcessTemplatesAction to return an error result.
  275. self._process_custom_roles(context)
  276. except Exception as err:
  277. LOG.exception("Error occurred while processing custom roles.")
  278. return actions.Result(error=six.text_type(err))
  279. template_name = plan_env.get('template')
  280. environments = plan_env.get('environments')
  281. env_paths = []
  282. temp_files = []
  283. template_object = os.path.join(swift.url, self.container,
  284. template_name)
  285. LOG.debug('Template: %s' % template_name)
  286. LOG.debug('Environments: %s' % environments)
  287. try:
  288. for env in environments:
  289. if env.get('path'):
  290. env_paths.append(os.path.join(swift.url, self.container,
  291. env['path']))
  292. elif env.get('data'):
  293. env_temp_file = _create_temp_file(env['data'])
  294. temp_files.append(env_temp_file)
  295. env_paths.append(env_temp_file)
  296. # create a dict to hold all user set params and merge
  297. # them in the appropriate order
  298. merged_params = {}
  299. # merge generated passwords into params first
  300. passwords = plan_env.get('passwords', {})
  301. merged_params.update(passwords)
  302. # derived parameters are merged before 'parameter defaults'
  303. # so that user-specified values can override the derived values.
  304. derived_params = plan_env.get('derived_parameters', {})
  305. merged_params.update(derived_params)
  306. # handle user set parameter values next in case a user has set
  307. # a new value for a password parameter
  308. params = plan_env.get('parameter_defaults', {})
  309. merged_params = template_utils.deep_update(merged_params, params)
  310. if merged_params:
  311. env_temp_file = _create_temp_file(
  312. {'parameter_defaults': merged_params})
  313. temp_files.append(env_temp_file)
  314. env_paths.append(env_temp_file)
  315. registry = plan_env.get('resource_registry', {})
  316. if registry:
  317. env_temp_file = _create_temp_file(
  318. {'resource_registry': registry})
  319. temp_files.append(env_temp_file)
  320. env_paths.append(env_temp_file)
  321. def _env_path_is_object(env_path):
  322. retval = env_path.startswith(swift.url)
  323. LOG.debug('_env_path_is_object %s: %s' % (env_path, retval))
  324. return retval
  325. def _object_request(method, url, token=context.auth_token):
  326. return requests.request(
  327. method, url, headers={'X-Auth-Token': token}).content
  328. template_files, template = template_utils.get_template_contents(
  329. template_object=template_object,
  330. object_request=_object_request)
  331. env_files, env = (
  332. template_utils.process_multiple_environments_and_files(
  333. env_paths=env_paths,
  334. env_path_is_object=_env_path_is_object,
  335. object_request=_object_request))
  336. except Exception as err:
  337. error_text = six.text_type(err)
  338. LOG.exception("Error occurred while processing plan files.")
  339. finally:
  340. # cleanup any local temp files
  341. for f in temp_files:
  342. os.remove(f)
  343. if error_text:
  344. return actions.Result(error=error_text)
  345. files = dict(list(template_files.items()) + list(env_files.items()))
  346. return {
  347. 'stack_name': self.container,
  348. 'template': template,
  349. 'environment': env,
  350. 'files': files
  351. }