Heat templates for deploying OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

583 lines
20KB

  1. #!/usr/bin/env python
  2. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  3. # not use this file except in compliance with the License. You may obtain
  4. # a copy of the License at
  5. #
  6. # http://www.apache.org/licenses/LICENSE-2.0
  7. #
  8. # Unless required by applicable law or agreed to in writing, software
  9. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11. # License for the specific language governing permissions and limitations
  12. # under the License.
  13. # Shell script tool to run puppet inside of the given container image.
  14. # Uses the config file at /var/lib/container-puppet/container-puppet.json
  15. # as a source for a JSON array of
  16. # [config_volume, puppet_tags, manifest, config_image, [volumes]] settings
  17. # that can be used to generate config files or run ad-hoc puppet modules
  18. # inside of a container.
  19. import glob
  20. import json
  21. import logging
  22. import multiprocessing
  23. import os
  24. import subprocess
  25. import sys
  26. import tempfile
  27. import time
  28. from paunch import runner as containers_runner
  29. def get_logger():
  30. """Return a logger object."""
  31. logger = logging.getLogger()
  32. ch = logging.StreamHandler(sys.stdout)
  33. if os.environ.get('DEBUG') in ['True', 'true']:
  34. logger.setLevel(logging.DEBUG)
  35. ch.setLevel(logging.DEBUG)
  36. else:
  37. logger.setLevel(logging.INFO)
  38. ch.setLevel(logging.INFO)
  39. formatter = logging.Formatter(
  40. '%(asctime)s %(levelname)s: %(process)s -- %(message)s'
  41. )
  42. ch.setFormatter(formatter)
  43. logger.addHandler(ch)
  44. return logger
  45. def local_subprocess_call(cmd, env=None):
  46. """General run method for subprocess calls.
  47. :param cmd: list
  48. returns: tuple
  49. """
  50. subproc = subprocess.Popen(
  51. cmd,
  52. stdout=subprocess.PIPE,
  53. stderr=subprocess.PIPE,
  54. universal_newlines=True,
  55. env=env
  56. )
  57. stdout, stderr = subproc.communicate()
  58. return stdout, stderr, subproc.returncode
  59. def pull_image(name):
  60. _, _, rc = local_subprocess_call(cmd=[CLI_CMD, 'inspect', name])
  61. if rc == 0:
  62. LOG.info('Image already exists: %s' % name)
  63. return
  64. retval = -1
  65. count = 0
  66. LOG.info('Pulling image: %s' % name)
  67. while retval != 0:
  68. count += 1
  69. stdout, stderr, retval = local_subprocess_call(
  70. cmd=[CLI_CMD, 'pull', name]
  71. )
  72. if retval != 0:
  73. time.sleep(3)
  74. LOG.warning('%s pull failed: %s' % (CONTAINER_CLI, stderr))
  75. LOG.warning('retrying pulling image: %s' % name)
  76. if count >= 5:
  77. LOG.error('Failed to pull image: %s' % name)
  78. break
  79. if stdout:
  80. LOG.debug(stdout)
  81. if stderr:
  82. LOG.debug(stderr)
  83. def match_config_volumes(prefix, config):
  84. # Match the mounted config volumes - we can't just use the
  85. # key as e.g "novacomute" consumes config-data/nova
  86. try:
  87. volumes = config.get('volumes', [])
  88. except AttributeError:
  89. LOG.error(
  90. 'Error fetching volumes. Prefix: %s - Config: %s' % (
  91. prefix,
  92. config
  93. )
  94. )
  95. raise
  96. return sorted([os.path.dirname(v.split(":")[0]) for v in volumes if
  97. v.startswith(prefix)])
  98. def get_config_hash(config_volume):
  99. hashfile = "%s.md5sum" % config_volume
  100. LOG.debug(
  101. "Looking for hashfile %s for config_volume %s" % (
  102. hashfile,
  103. config_volume
  104. )
  105. )
  106. hash_data = None
  107. if os.path.isfile(hashfile):
  108. LOG.debug(
  109. "Got hashfile %s for config_volume %s" % (
  110. hashfile,
  111. config_volume
  112. )
  113. )
  114. with open(hashfile) as f:
  115. hash_data = f.read().rstrip()
  116. return hash_data
  117. def rm_container(name):
  118. if os.environ.get('SHOW_DIFF', None):
  119. LOG.info('Diffing container: %s' % name)
  120. stdout, stderr, retval = local_subprocess_call(
  121. cmd=[CLI_CMD, 'diff', name]
  122. )
  123. if stdout:
  124. LOG.debug(stdout)
  125. if stderr:
  126. LOG.debug(stderr)
  127. def rm_process_call(rm_cli_cmd):
  128. stdout, stderr, retval = local_subprocess_call(
  129. cmd=rm_cli_cmd)
  130. if stdout:
  131. LOG.debug(stdout)
  132. if stderr and 'Error response from daemon' in stderr:
  133. LOG.debug(stderr)
  134. LOG.info('Removing container: %s' % name)
  135. rm_cli_cmd = [CLI_CMD, 'rm']
  136. rm_cli_cmd.append(name)
  137. rm_process_call(rm_cli_cmd)
  138. # --storage is used as a mitigation of
  139. # https://github.com/containers/libpod/issues/3906
  140. # Also look https://bugzilla.redhat.com/show_bug.cgi?id=1747885
  141. if CONTAINER_CLI == 'podman':
  142. rm_storage_cli_cmd = [CLI_CMD, 'rm', '--storage']
  143. rm_storage_cli_cmd.append(name)
  144. rm_process_call(rm_storage_cli_cmd)
  145. def mp_puppet_config(*args):
  146. (
  147. config_volume,
  148. puppet_tags,
  149. manifest,
  150. config_image,
  151. volumes,
  152. privileged,
  153. check_mode,
  154. keep_container
  155. ) = args[0]
  156. LOG.info('Starting configuration of %s using image %s' %
  157. (config_volume, config_image))
  158. LOG.debug('config_volume %s' % config_volume)
  159. LOG.debug('puppet_tags %s' % puppet_tags)
  160. LOG.debug('manifest %s' % manifest)
  161. LOG.debug('config_image %s' % config_image)
  162. LOG.debug('volumes %s' % volumes)
  163. LOG.debug('privileged %s' % privileged)
  164. LOG.debug('check_mode %s' % check_mode)
  165. LOG.debug('keep_container %s' % keep_container)
  166. with tempfile.NamedTemporaryFile() as tmp_man:
  167. with open(tmp_man.name, 'w') as man_file:
  168. man_file.write('include ::tripleo::packages\n')
  169. man_file.write(manifest)
  170. uname = RUNNER.unique_container_name(
  171. 'container-puppet-%s' % config_volume
  172. )
  173. rm_container(uname)
  174. pull_image(config_image)
  175. common_dcmd = [
  176. CLI_CMD,
  177. 'run',
  178. '--user',
  179. 'root',
  180. '--name',
  181. uname,
  182. '--env',
  183. 'PUPPET_TAGS=%s' % puppet_tags,
  184. '--env',
  185. 'NAME=%s' % config_volume,
  186. '--env',
  187. 'HOSTNAME=%s' % os.environ.get('SHORT_HOSTNAME'),
  188. '--env',
  189. 'NO_ARCHIVE=%s' % os.environ.get('NO_ARCHIVE', ''),
  190. '--env',
  191. 'STEP=%s' % os.environ.get('STEP', '6'),
  192. '--env',
  193. 'NET_HOST=%s' % os.environ.get('NET_HOST', 'false'),
  194. '--env',
  195. 'DEBUG=%s' % os.environ.get('DEBUG', 'false'),
  196. '--volume',
  197. '/etc/localtime:/etc/localtime:ro',
  198. '--volume',
  199. '%s:/etc/config.pp:ro' % tmp_man.name,
  200. '--volume',
  201. '/etc/puppet/:/tmp/puppet-etc/:ro',
  202. # OpenSSL trusted CA injection
  203. '--volume',
  204. '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
  205. '--volume',
  206. '/etc/pki/tls/certs/ca-bundle.crt:'
  207. '/etc/pki/tls/certs/ca-bundle.crt:ro',
  208. '--volume',
  209. '/etc/pki/tls/certs/ca-bundle.trust.crt:'
  210. '/etc/pki/tls/certs/ca-bundle.trust.crt:ro',
  211. '--volume',
  212. '/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro',
  213. '--volume',
  214. '%s:/var/lib/config-data/:rw' % CONFIG_VOLUME_PREFIX,
  215. # facter caching
  216. '--volume',
  217. '/var/lib/container-puppet/puppetlabs/facter.conf:'
  218. '/etc/puppetlabs/facter/facter.conf:ro',
  219. '--volume',
  220. '/var/lib/container-puppet/puppetlabs/:/opt/puppetlabs/:ro',
  221. # Syslog socket for puppet logs
  222. '--volume', '/dev/log:/dev/log:rw'
  223. ]
  224. # Remove container by default after the run
  225. # This should mitigate the "ghost container" issue described here
  226. # https://bugzilla.redhat.com/show_bug.cgi?id=1747885
  227. # https://bugs.launchpad.net/tripleo/+bug/1840691
  228. if not keep_container:
  229. common_dcmd.append('--rm')
  230. if privileged:
  231. common_dcmd.append('--privileged')
  232. if CONTAINER_CLI == 'podman':
  233. log_path = os.path.join(CONTAINER_LOG_STDOUT_PATH, uname)
  234. logging = ['--log-driver', 'k8s-file',
  235. '--log-opt',
  236. 'path=%s.log' % log_path]
  237. common_dcmd.extend(logging)
  238. elif CONTAINER_CLI == 'docker':
  239. # NOTE(flaper87): Always copy the DOCKER_* environment variables as
  240. # they contain the access data for the docker daemon.
  241. for k in os.environ.keys():
  242. if k.startswith('DOCKER'):
  243. ENV[k] = os.environ.get(k)
  244. common_dcmd += CLI_DCMD
  245. if CHECK_MODE:
  246. common_dcmd.extend([
  247. '--volume',
  248. '/etc/puppet/check-mode:/tmp/puppet-check-mode:ro'])
  249. for volume in volumes:
  250. if volume:
  251. common_dcmd.extend(['--volume', volume])
  252. common_dcmd.extend(['--entrypoint', SH_SCRIPT])
  253. if os.environ.get('NET_HOST', 'false') == 'true':
  254. LOG.debug('NET_HOST enabled')
  255. common_dcmd.extend(['--net', 'host', '--volume',
  256. '/etc/hosts:/etc/hosts:ro'])
  257. else:
  258. LOG.debug('running without containers Networking')
  259. common_dcmd.extend(['--net', 'none'])
  260. # script injection as the last mount to make sure it's accessible
  261. # https://github.com/containers/libpod/issues/1844
  262. common_dcmd.extend(['--volume', '%s:%s:ro' % (SH_SCRIPT, SH_SCRIPT)])
  263. common_dcmd.append(config_image)
  264. # https://github.com/containers/libpod/issues/1844
  265. # This block will run "CONTAINER_CLI" run 5 times before to fail.
  266. retval = -1
  267. count = 0
  268. LOG.debug(
  269. 'Running %s command: %s' % (
  270. CONTAINER_CLI,
  271. ' '.join(common_dcmd)
  272. )
  273. )
  274. while count < 3:
  275. count += 1
  276. stdout, stderr, retval = local_subprocess_call(
  277. cmd=common_dcmd,
  278. env=ENV
  279. )
  280. # puppet with --detailed-exitcodes will return 0 for success and
  281. # no changes and 2 for success and resource changes. Other
  282. # numbers are failures
  283. if retval in [0, 2]:
  284. if stdout:
  285. LOG.debug('%s run succeeded: %s' % (common_dcmd, stdout))
  286. if stderr:
  287. LOG.warning(stderr)
  288. # only delete successful runs, for debugging
  289. rm_container(uname)
  290. break
  291. time.sleep(3)
  292. LOG.error(
  293. '%s run failed after %s attempt(s): %s' % (
  294. common_dcmd,
  295. stderr,
  296. count
  297. )
  298. )
  299. LOG.warning('Retrying running container: %s' % config_volume)
  300. else:
  301. if stdout:
  302. LOG.debug(stdout)
  303. if stderr:
  304. LOG.debug(stderr)
  305. LOG.error('Failed running container for %s' % config_volume)
  306. LOG.info(
  307. 'Finished processing puppet configs for %s' % (
  308. config_volume
  309. )
  310. )
  311. return retval
  312. def infile_processing(infiles):
  313. for infile in infiles:
  314. # If the JSON is already hashed, we'll skip it; and a new hashed file will
  315. # be created if config changed.
  316. if 'hashed' in infile:
  317. LOG.debug('%s skipped, already hashed' % infile)
  318. continue
  319. with open(infile) as f:
  320. infile_data = json.load(f)
  321. # if the contents of the file is None, we need should just create an empty
  322. # data set see LP#1828295
  323. if not infile_data:
  324. infile_data = {}
  325. c_name = os.path.splitext(os.path.basename(infile))[0]
  326. config_volumes = match_config_volumes(
  327. CONFIG_VOLUME_PREFIX,
  328. infile_data
  329. )
  330. config_hashes = [
  331. get_config_hash(volume_path) for volume_path in config_volumes
  332. ]
  333. config_hashes = filter(None, config_hashes)
  334. config_hash = '-'.join(config_hashes)
  335. if config_hash:
  336. LOG.debug(
  337. "Updating config hash for %s, config_volume=%s hash=%s" % (
  338. c_name,
  339. config_volume,
  340. config_hash
  341. )
  342. )
  343. # When python 27 support is removed, we will be able to use:
  344. # z = {**x, **y} to merge the dicts.
  345. if infile_data.get('environment', None) is None:
  346. infile_data['environment'] = {}
  347. infile_data['environment'].update(
  348. {'TRIPLEO_CONFIG_HASH': config_hash}
  349. )
  350. outfile = os.path.join(
  351. os.path.dirname(
  352. infile
  353. ), "hashed-" + os.path.basename(infile)
  354. )
  355. with open(outfile, 'w') as out_f:
  356. os.chmod(out_f.name, 0o600)
  357. json.dump(infile_data, out_f, indent=2)
  358. if __name__ == '__main__':
  359. PUPPETS = (
  360. '/usr/share/openstack-puppet/modules/:'
  361. '/usr/share/openstack-puppet/modules/:ro'
  362. )
  363. SH_SCRIPT = '/var/lib/container-puppet/container-puppet.sh'
  364. CONTAINER_CLI = os.environ.get('CONTAINER_CLI', 'podman')
  365. CONTAINER_LOG_STDOUT_PATH = os.environ.get(
  366. 'CONTAINER_LOG_STDOUT_PATH',
  367. '/var/log/containers/stdouts'
  368. )
  369. CLI_CMD = '/usr/bin/' + CONTAINER_CLI
  370. LOG = get_logger()
  371. LOG.info('Running container-puppet')
  372. CONFIG_VOLUME_PREFIX = os.path.abspath(
  373. os.environ.get(
  374. 'CONFIG_VOLUME_PREFIX',
  375. '/var/lib/config-data'
  376. )
  377. )
  378. CHECK_MODE = int(os.environ.get('CHECK_MODE', 0))
  379. LOG.debug('CHECK_MODE: %s' % CHECK_MODE)
  380. if CONTAINER_CLI == 'docker':
  381. CLI_DCMD = ['--volume', PUPPETS]
  382. ENV = {}
  383. RUNNER = containers_runner.DockerRunner(
  384. 'container-puppet',
  385. cont_cmd='docker',
  386. log=LOG
  387. )
  388. elif CONTAINER_CLI == 'podman':
  389. # podman doesn't allow relabeling content in /usr and
  390. # doesn't support named volumes
  391. CLI_DCMD = [
  392. '--security-opt',
  393. 'label=disable',
  394. '--volume',
  395. PUPPETS
  396. ]
  397. # podman need to find dependent binaries that are in environment
  398. ENV = {'PATH': os.environ['PATH']}
  399. RUNNER = containers_runner.PodmanRunner(
  400. 'container-puppet',
  401. cont_cmd='podman',
  402. log=LOG
  403. )
  404. else:
  405. LOG.error('Invalid CONTAINER_CLI: %s' % CONTAINER_CLI)
  406. raise SystemExit()
  407. config_file = os.environ.get(
  408. 'CONFIG',
  409. '/var/lib/container-puppet/container-puppet.json'
  410. )
  411. LOG.debug('CONFIG: %s' % config_file)
  412. # If specified, only this config_volume will be used
  413. CONFIG_VOLUME_ONLY = os.environ.get('CONFIG_VOLUME', None)
  414. with open(config_file) as f:
  415. JSON_DATA = json.load(f)
  416. # To save time we support configuring 'shared' services at the same
  417. # time. For example configuring all of the heat services
  418. # in a single container pass makes sense and will save some time.
  419. # To support this we merge shared settings together here.
  420. #
  421. # We key off of config_volume as this should be the same for a
  422. # given group of services. We are also now specifying the container
  423. # in which the services should be configured. This should match
  424. # in all instances where the volume name is also the same.
  425. CONFIGS = {}
  426. for service in (JSON_DATA or []):
  427. if service is None:
  428. continue
  429. if isinstance(service, dict):
  430. service = [
  431. service.get('config_volume'),
  432. service.get('puppet_tags'),
  433. service.get('step_config'),
  434. service.get('config_image'),
  435. service.get('volumes', []),
  436. service.get('privileged', False),
  437. ]
  438. CONFIG_VOLUME = service[0] or ''
  439. PUPPET_TAGS = service[1] or ''
  440. MANIFEST = service[2] or ''
  441. CONFIG_IMAGE = service[3] or ''
  442. VOLUMES = service[4] if len(service) > 4 else []
  443. if not MANIFEST or not CONFIG_IMAGE:
  444. continue
  445. LOG.debug('config_volume %s' % CONFIG_VOLUME)
  446. LOG.debug('puppet_tags %s' % PUPPET_TAGS)
  447. LOG.debug('manifest %s' % MANIFEST)
  448. LOG.debug('config_image %s' % CONFIG_IMAGE)
  449. LOG.debug('volumes %s' % VOLUMES)
  450. LOG.debug('privileged %s' % service[5] if len(service) > 5 else False)
  451. # We key off of config volume for all configs.
  452. if CONFIG_VOLUME in CONFIGS:
  453. # Append puppet tags and manifest.
  454. LOG.debug("Existing service, appending puppet tags and manifest")
  455. if PUPPET_TAGS:
  456. CONFIGS[CONFIG_VOLUME][1] = '%s,%s' % (
  457. CONFIGS[CONFIG_VOLUME][1],
  458. PUPPET_TAGS
  459. )
  460. if MANIFEST:
  461. CONFIGS[CONFIG_VOLUME][2] = '%s\n%s' % (
  462. CONFIGS[CONFIG_VOLUME][2],
  463. MANIFEST
  464. )
  465. if CONFIGS[CONFIG_VOLUME][3] != CONFIG_IMAGE:
  466. LOG.warning("Config containers do not match even though"
  467. " shared volumes are the same!")
  468. if VOLUMES:
  469. CONFIGS[CONFIG_VOLUME][4].extend(VOLUMES)
  470. else:
  471. if not CONFIG_VOLUME_ONLY or (CONFIG_VOLUME_ONLY == CONFIG_VOLUME):
  472. LOG.debug("Adding new service")
  473. CONFIGS[CONFIG_VOLUME] = service
  474. else:
  475. LOG.debug(
  476. "Ignoring %s due to $CONFIG_VOLUME=%s" % (
  477. CONFIG_VOLUME,
  478. CONFIG_VOLUME_ONLY
  479. )
  480. )
  481. LOG.info('Service compilation completed.')
  482. # Holds all the information for each process to consume.
  483. # Instead of starting them all linearly we run them using a process
  484. # pool. This creates a list of arguments for the above function
  485. # to consume.
  486. PROCESS_MAP = []
  487. for config_volume in CONFIGS:
  488. SERVICE = CONFIGS[config_volume]
  489. PUPPET_TAGS = SERVICE[1] or ''
  490. if PUPPET_TAGS:
  491. PUPPET_TAGS = "file,file_line,concat,augeas,cron,%s" % PUPPET_TAGS
  492. else:
  493. PUPPET_TAGS = "file,file_line,concat,augeas,cron"
  494. PROCESS_ITEM = [
  495. config_volume,
  496. PUPPET_TAGS,
  497. SERVICE[2] or '',
  498. SERVICE[3] or '',
  499. SERVICE[4] if len(SERVICE) > 4 else [],
  500. SERVICE[5] if len(SERVICE) > 5 else False,
  501. CHECK_MODE,
  502. SERVICE[6] if len(SERVICE) > 6 else False
  503. ]
  504. PROCESS_MAP.append(PROCESS_ITEM)
  505. LOG.debug('- %s' % PROCESS_ITEM)
  506. # Fire off processes to perform each configuration. Defaults
  507. # to the number of CPUs on the system.
  508. PROCESS = multiprocessing.Pool(int(os.environ.get('PROCESS_COUNT', 2)))
  509. RETURNCODES = list(PROCESS.map(mp_puppet_config, PROCESS_MAP))
  510. CONFIG_VOLUMES = [pm[0] for pm in PROCESS_MAP]
  511. SUCCESS = True
  512. for returncode, config_volume in zip(RETURNCODES, CONFIG_VOLUMES):
  513. if returncode not in [0, 2]:
  514. LOG.error('ERROR configuring %s' % config_volume)
  515. SUCCESS = False
  516. # Update the startup configs with the config hash we generated above
  517. STARTUP_CONFIGS = os.environ.get(
  518. 'STARTUP_CONFIG_PATTERN',
  519. '/var/lib/tripleo-config/docker-container-startup-config-step_*.json'
  520. )
  521. LOG.debug('STARTUP_CONFIG_PATTERN: %s' % STARTUP_CONFIGS)
  522. # Run infile processing
  523. infile_processing(infiles=glob.glob(STARTUP_CONFIGS))
  524. if not SUCCESS:
  525. raise SystemExit()