A Python library for code common to TripleO CLI and TripleO UI.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

kolla_builder.py 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495
  1. # Copyright 2017 Red Hat, Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. #
  15. import jinja2
  16. import logging
  17. import os
  18. import re
  19. import subprocess
  20. import sys
  21. import tempfile
  22. import time
  23. import yaml
  24. from tripleo_common.image import base
  25. from tripleo_common.image import image_uploader
  26. CONTAINER_IMAGE_PREPARE_PARAM_STR = None
  27. CONTAINER_IMAGE_PREPARE_PARAM = None
  28. CONTAINER_IMAGES_DEFAULTS = None
  29. def init_prepare_defaults(defaults_file):
  30. global CONTAINER_IMAGE_PREPARE_PARAM_STR
  31. with open(defaults_file) as f:
  32. CONTAINER_IMAGE_PREPARE_PARAM_STR = f.read()
  33. global CONTAINER_IMAGE_PREPARE_PARAM
  34. p = yaml.safe_load(CONTAINER_IMAGE_PREPARE_PARAM_STR)
  35. CONTAINER_IMAGE_PREPARE_PARAM = p[
  36. 'parameter_defaults']['ContainerImagePrepare']
  37. global CONTAINER_IMAGES_DEFAULTS
  38. CONTAINER_IMAGES_DEFAULTS = CONTAINER_IMAGE_PREPARE_PARAM[0]['set']
  39. DEFAULT_TEMPLATE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
  40. 'container-images',
  41. 'overcloud_containers.yaml.j2')
  42. DEFAULT_PREPARE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
  43. 'container-images',
  44. 'container_image_prepare_defaults.yaml')
  45. if os.path.isfile(DEFAULT_PREPARE_FILE):
  46. init_prepare_defaults(DEFAULT_PREPARE_FILE)
  47. def get_enabled_services(environment, roles_data):
  48. """Build list of enabled services
  49. :param environment: Heat environment for deployment
  50. :param roles_data: Roles file data used to filter services
  51. :returns: set of resource types representing enabled services
  52. """
  53. enabled_services = set()
  54. parameter_defaults = environment.get('parameter_defaults', {})
  55. for role in roles_data:
  56. count = parameter_defaults.get('%sCount' % role['name'],
  57. role.get('CountDefault', 0))
  58. if count > 0:
  59. enabled_services.update(
  60. parameter_defaults.get('%sServices' % role['name'],
  61. role.get('ServicesDefault', [])))
  62. return enabled_services
  63. def build_service_filter(environment, roles_data):
  64. """Build list of containerized services
  65. :param environment: Heat environment for deployment
  66. :param roles_data: Roles file data used to filter services
  67. :returns: set of resource types representing containerized services
  68. """
  69. if not roles_data:
  70. return None
  71. enabled_services = get_enabled_services(environment, roles_data)
  72. resource_registry = environment.get('resource_registry')
  73. if not resource_registry:
  74. # no way to tell which services are non-containerized, so
  75. # filter by enabled services
  76. return enabled_services
  77. for service, env_path in environment.get('resource_registry', {}).items():
  78. if service not in enabled_services:
  79. continue
  80. if env_path == 'OS::Heat::None':
  81. enabled_services.remove(service)
  82. if '/puppet/services' in env_path:
  83. enabled_services.remove(service)
  84. return enabled_services
  85. def set_neutron_driver(pd, mapping_args):
  86. """Set the neutron_driver images variable based on parameters
  87. :param pd: Parameter defaults from the environment
  88. :param mapping_args: Dict to set neutron_driver value on
  89. """
  90. if not pd or 'NeutronMechanismDrivers' not in pd:
  91. # we should set default neutron driver
  92. mapping_args['neutron_driver'] = 'ovn'
  93. return
  94. nmd = pd['NeutronMechanismDrivers']
  95. if 'opendaylight_v2' in nmd:
  96. mapping_args['neutron_driver'] = 'odl'
  97. elif 'ovn' in nmd:
  98. mapping_args['neutron_driver'] = 'ovn'
  99. else:
  100. mapping_args['neutron_driver'] = 'other'
  101. def container_images_prepare_multi(environment, roles_data, dry_run=False,
  102. cleanup=image_uploader.CLEANUP_FULL):
  103. """Perform multiple container image prepares and merge result
  104. Given the full heat environment and roles data, perform multiple image
  105. prepare operations. The data to drive the multiple prepares is taken from
  106. the ContainerImagePrepare parameter in the provided environment. If
  107. push_destination is specified, uploads will be performed during the
  108. preparation.
  109. :param environment: Heat environment for deployment
  110. :param roles_data: Roles file data used to filter services
  111. :returns: dict containing merged container image parameters from all
  112. prepare operations
  113. """
  114. pd = environment.get('parameter_defaults', {})
  115. cip = pd.get('ContainerImagePrepare')
  116. if not cip:
  117. return
  118. mirrors = {}
  119. mirror = pd.get('DockerRegistryMirror')
  120. if mirror:
  121. mirrors['docker.io'] = mirror
  122. env_params = {}
  123. service_filter = build_service_filter(environment, roles_data)
  124. for cip_entry in cip:
  125. mapping_args = cip_entry.get('set', {})
  126. set_neutron_driver(pd, mapping_args)
  127. push_destination = cip_entry.get('push_destination')
  128. # use the configured registry IP as the discovered registry
  129. # if it is available
  130. if push_destination and isinstance(push_destination, bool):
  131. local_registry_ip = pd.get('LocalContainerRegistry')
  132. if local_registry_ip:
  133. push_destination = '%s:8787' % local_registry_ip
  134. pull_source = cip_entry.get('pull_source')
  135. modify_role = cip_entry.get('modify_role')
  136. modify_vars = cip_entry.get('modify_vars')
  137. modify_only_with_labels = cip_entry.get('modify_only_with_labels')
  138. modify_append_tag = cip_entry.get('modify_append_tag',
  139. time.strftime(
  140. '-modified-%Y%m%d%H%M%S'))
  141. prepare_data = container_images_prepare(
  142. excludes=cip_entry.get('excludes'),
  143. includes=cip_entry.get('includes'),
  144. service_filter=service_filter,
  145. pull_source=pull_source,
  146. push_destination=push_destination,
  147. mapping_args=mapping_args,
  148. output_env_file='image_params',
  149. output_images_file='upload_data',
  150. tag_from_label=cip_entry.get('tag_from_label'),
  151. append_tag=modify_append_tag,
  152. modify_role=modify_role,
  153. modify_vars=modify_vars,
  154. modify_only_with_labels=modify_only_with_labels,
  155. mirrors=mirrors
  156. )
  157. env_params.update(prepare_data['image_params'])
  158. if push_destination or pull_source or modify_role:
  159. with tempfile.NamedTemporaryFile(mode='w') as f:
  160. yaml.safe_dump({
  161. 'container_images': prepare_data['upload_data']
  162. }, f)
  163. uploader = image_uploader.ImageUploadManager(
  164. [f.name],
  165. dry_run=dry_run,
  166. cleanup=cleanup,
  167. mirrors=mirrors
  168. )
  169. uploader.upload()
  170. return env_params
  171. def container_images_prepare_defaults():
  172. """Return default dict for prepare substitutions
  173. This can be used as the mapping_args argument to the
  174. container_images_prepare function to get the same result as not specifying
  175. any mapping_args.
  176. """
  177. return KollaImageBuilder.container_images_template_inputs()
  178. def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
  179. excludes=None, includes=None, service_filter=None,
  180. pull_source=None, push_destination=None,
  181. mapping_args=None, output_env_file=None,
  182. output_images_file=None, tag_from_label=None,
  183. append_tag=None, modify_role=None,
  184. modify_vars=None, modify_only_with_labels=None,
  185. mirrors=None):
  186. """Perform container image preparation
  187. :param template_file: path to Jinja2 file containing all image entries
  188. :param excludes: list of image name substrings to use for exclude filter
  189. :param includes: list of image name substrings, at least one must match.
  190. All excludes are ignored if includes is specified.
  191. :param service_filter: set of heat resource types for containerized
  192. services to filter by. Disable by passing None.
  193. :param pull_source: DEPRECATED namespace for pulling during image uploads
  194. :param push_destination: namespace for pushing during image uploads. When
  195. specified the image parameters will use this
  196. namespace too.
  197. :param mapping_args: dict containing substitutions for template file. See
  198. CONTAINER_IMAGES_DEFAULTS for expected keys.
  199. :param output_env_file: key to use for heat environment parameter data
  200. :param output_images_file: key to use for image upload data
  201. :param tag_from_label: string when set will trigger tag discovery on every
  202. image
  203. :param append_tag: string to append to the tag for the destination
  204. image
  205. :param modify_role: string of ansible role name to run during upload before
  206. the push to destination
  207. :param modify_vars: dict of variables to pass to modify_role
  208. :param modify_only_with_labels: only modify the container images with the
  209. given labels
  210. :param mirrors: dict of registry netloc values to mirror urls
  211. :returns: dict with entries for the supplied output_env_file or
  212. output_images_file
  213. """
  214. if mapping_args is None:
  215. mapping_args = {}
  216. def ffunc(entry):
  217. imagename = entry.get('imagename', '')
  218. if service_filter is not None:
  219. # check the entry is for a service being deployed
  220. image_services = set(entry.get('services', []))
  221. if not image_services.intersection(service_filter):
  222. return None
  223. if includes:
  224. for p in includes:
  225. if re.search(p, imagename):
  226. return entry
  227. return None
  228. if excludes:
  229. for p in excludes:
  230. if re.search(p, imagename):
  231. return None
  232. return entry
  233. builder = KollaImageBuilder([template_file])
  234. result = builder.container_images_from_template(
  235. filter=ffunc, **mapping_args)
  236. manager = image_uploader.ImageUploadManager(mirrors=mirrors)
  237. uploader = manager.uploader('python')
  238. images = [i.get('imagename', '') for i in result]
  239. if tag_from_label:
  240. image_version_tags = uploader.discover_image_tags(
  241. images, tag_from_label)
  242. for entry in result:
  243. imagename = entry.get('imagename', '')
  244. image_no_tag = imagename.rpartition(':')[0]
  245. if image_no_tag in image_version_tags:
  246. entry['imagename'] = '%s:%s' % (
  247. image_no_tag, image_version_tags[image_no_tag])
  248. if modify_only_with_labels:
  249. images_with_labels = uploader.filter_images_with_labels(
  250. images, modify_only_with_labels)
  251. params = {}
  252. modify_append_tag = append_tag
  253. for entry in result:
  254. imagename = entry.get('imagename', '')
  255. append_tag = ''
  256. if modify_role and (
  257. (not modify_only_with_labels) or
  258. imagename in images_with_labels):
  259. entry['modify_role'] = modify_role
  260. if modify_append_tag:
  261. entry['modify_append_tag'] = modify_append_tag
  262. append_tag = modify_append_tag
  263. if modify_vars:
  264. entry['modify_vars'] = modify_vars
  265. if pull_source:
  266. entry['pull_source'] = pull_source
  267. if push_destination:
  268. # substitute discovered registry if push_destination is set to true
  269. if isinstance(push_destination, bool):
  270. push_destination = image_uploader.get_undercloud_registry()
  271. entry['push_destination'] = push_destination
  272. # replace the host portion of the imagename with the
  273. # push_destination, since that is where they will be uploaded to
  274. image = imagename.partition('/')[2]
  275. imagename = '/'.join((push_destination, image))
  276. if 'params' in entry:
  277. for p in entry.pop('params'):
  278. params[p] = imagename + append_tag
  279. if 'services' in entry:
  280. del(entry['services'])
  281. params.update(
  282. detect_insecure_registries(params))
  283. return_data = {}
  284. if output_env_file:
  285. return_data[output_env_file] = params
  286. if output_images_file:
  287. return_data[output_images_file] = result
  288. return return_data
  289. def detect_insecure_registries(params):
  290. """Detect insecure registries in image parameters
  291. :param params: dict of container image parameters
  292. :returns: dict containing DockerInsecureRegistryAddress parameter to be
  293. merged into other parameters
  294. """
  295. insecure = set()
  296. uploader = image_uploader.ImageUploadManager().uploader('python')
  297. for image in params.values():
  298. host = image.split('/')[0]
  299. if uploader.is_insecure_registry(host):
  300. insecure.add(host)
  301. if not insecure:
  302. return {}
  303. return {'DockerInsecureRegistryAddress': sorted(insecure)}
  304. class KollaImageBuilder(base.BaseImageManager):
  305. """Build images using kolla-build"""
  306. logger = logging.getLogger(__name__ + '.KollaImageBuilder')
  307. handler = logging.StreamHandler(sys.stdout)
  308. @staticmethod
  309. def imagename_to_regex(imagename):
  310. if not imagename:
  311. return
  312. # remove any namespace from the start
  313. imagename = imagename.split('/')[-1]
  314. # remove any tag from the end
  315. imagename = imagename.split(':')[0]
  316. # remove supported base names from the start
  317. imagename = re.sub(r'^(centos|rhel)-', '', imagename)
  318. # remove install_type from the start
  319. imagename = re.sub(r'^(binary|source|rdo|rhos)-', '', imagename)
  320. # what results should be acceptable as a regex to build one image
  321. return imagename
  322. @staticmethod
  323. def container_images_template_inputs(**kwargs):
  324. '''Build the template mapping from defaults and keyword arguments.
  325. Defaults in CONTAINER_IMAGES_DEFAULTS are combined with keyword
  326. argments to return a dict that can be used to render the container
  327. images template. Any set values for name_prefix and name_suffix are
  328. hyphenated appropriately.
  329. '''
  330. mapping = dict(kwargs)
  331. if CONTAINER_IMAGES_DEFAULTS is None:
  332. return
  333. for k, v in CONTAINER_IMAGES_DEFAULTS.items():
  334. mapping.setdefault(k, v)
  335. np = mapping['name_prefix']
  336. if np and not np.endswith('-'):
  337. mapping['name_prefix'] = np + '-'
  338. ns = mapping['name_suffix']
  339. if ns and not ns.startswith('-'):
  340. mapping['name_suffix'] = '-' + ns
  341. return mapping
  342. def container_images_from_template(self, filter=None, **kwargs):
  343. '''Build container_images data from container_images_template.
  344. Any supplied keyword arguments are used for the substitution mapping to
  345. transform the data in the config file container_images_template
  346. section.
  347. The resulting data resembles a config file which contains a valid
  348. populated container_images section.
  349. If a function is passed to the filter argument, this will be used to
  350. modify the entry after substitution. If the filter function returns
  351. None then the entry will not be added to the resulting list.
  352. Defaults are applied so that when no arguments are provided.
  353. '''
  354. mapping = self.container_images_template_inputs(**kwargs)
  355. result = []
  356. if len(self.config_files) != 1:
  357. raise ValueError('A single config file must be specified')
  358. config_file = self.config_files[0]
  359. with open(config_file) as cf:
  360. template = jinja2.Template(cf.read())
  361. rendered = template.render(mapping)
  362. rendered_dict = yaml.safe_load(rendered)
  363. for i in rendered_dict[self.CONTAINER_IMAGES_TEMPLATE]:
  364. entry = dict(i)
  365. if filter:
  366. entry = filter(entry)
  367. if entry is not None:
  368. result.append(entry)
  369. return result
  370. def build_images(self, kolla_config_files=None, excludes=[],
  371. template_only=False, kolla_tmp_dir=None):
  372. cmd = ['kolla-build']
  373. if kolla_config_files:
  374. for f in kolla_config_files:
  375. cmd.append('--config-file')
  376. cmd.append(f)
  377. if len(self.config_files) == 0:
  378. self.config_files = [DEFAULT_TEMPLATE_FILE]
  379. container_images = self.container_images_from_template()
  380. else:
  381. container_images = self.load_config_files(self.CONTAINER_IMAGES) \
  382. or []
  383. container_images.sort(key=lambda i: i.get('imagename'))
  384. for i in container_images:
  385. # Do not attempt to build containers that are not from kolla or
  386. # are in our exclude list
  387. if not i.get('image_source', '') == 'kolla':
  388. continue
  389. image = self.imagename_to_regex(i.get('imagename'))
  390. # Make sure the image was properly parsed and not purposely skipped
  391. if image and image not in excludes:
  392. cmd.append(image)
  393. if template_only:
  394. # build the dep list cmd line
  395. cmd_deps = list(cmd)
  396. cmd_deps.append('--list-dependencies')
  397. # build the template only cmd line
  398. cmd.append('--template-only')
  399. cmd.append('--work-dir')
  400. cmd.append(kolla_tmp_dir)
  401. self.logger.info('Running %s' % ' '.join(cmd))
  402. env = os.environ.copy()
  403. process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
  404. universal_newlines=True)
  405. out, err = process.communicate()
  406. if process.returncode != 0:
  407. raise subprocess.CalledProcessError(process.returncode, cmd, err)
  408. if template_only:
  409. self.logger.info('Running %s' % ' '.join(cmd_deps))
  410. env = os.environ.copy()
  411. process = subprocess.Popen(cmd_deps, env=env,
  412. stdout=subprocess.PIPE,
  413. stderr=subprocess.PIPE,
  414. universal_newlines=True)
  415. out, err = process.communicate()
  416. if process.returncode != 0:
  417. raise subprocess.CalledProcessError(process.returncode,
  418. cmd_deps, err)
  419. return out