#!/usr/bin/python # -*- coding: utf-8 -*- # Copyright 2019 Red Hat, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. __metaclass__ = type from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.parsing.convert_bool import boolean from datetime import datetime import base64 import copy import fnmatch import json import os import shutil import tempfile import yaml ANSIBLE_METADATA = { 'metadata_version': '1.1', 'status': ['preview'], 'supported_by': 'community' } DOCUMENTATION = """ --- module: container_puppet_config author: - "Emilien Macchi (@EmilienM)" version_added: '2.9' short_description: Generate puppet containers configs notes: [] description: - Generate puppet containers configs requirements: - None options: no_archive: description: - Disables config-data archiving type: bool default: True check_mode: description: - Ansible check mode is enabled type: bool default: False config_vol_prefix: description: - Config volume prefix type: str default: '/var/lib/config-data' debug: description: - Enable debug type: bool default: False net_host: description: - Using host network type: bool default: True puppet_config: description: Path to the puppet configs type: str default: "" short_hostname: description: - Short hostname type: str default: "" step: description: - Step number default: 6 type: int update_config_hash_only: description: - When set to True, the module will only inspect for new config hashes in config_vol_prefix and make sure the container-startup-configs are updated with these hashes. This is useful to execute before we manage the startup containers, so they will be restarted if needed (e.g. new config has been applied, container needs restart). type: bool default: False """ EXAMPLES = """ - name: Generate puppet container config for step 1 container_puppet_config: step: 1 puppet-config: /var/lib/container-puppet/container-puppet.json short_hostname: "{{ ansible_hostname }}" update_config_hash_only: false - name: Update config hashes for container startup configs container_puppet_config: update_config_hash_only: true """ CONTAINER_PUPPET_CONFIG = '/var/lib/tripleo-config/container-puppet-config' CONTAINER_STARTUP_CONFIG = '/var/lib/tripleo-config/container-startup-config' CONTAINER_ENTRYPOINT = '/var/lib/container-puppet/container-puppet.sh' class ContainerPuppetManager: """Notes about this module. It will generate container config that will be consumed by the tripleo-container-manage role that is using podman_container module. """ def __init__(self, module, results): super(ContainerPuppetManager, self).__init__() self.module = module self.results = results # parse args args = self.module.params # Set parameters puppet_config = args['puppet_config'] update_config_hash_only = args['update_config_hash_only'] self.config_vol_prefix = args['config_vol_prefix'] if not update_config_hash_only: data = json.loads(self._slurp(puppet_config)) self.step = args['step'] self.net_host = args['net_host'] self.debug = args['debug'] self.check = args['check_mode'] self.no_archive = args['no_archive'] self.hostname = args['short_hostname'] config_path = os.path.join(CONTAINER_PUPPET_CONFIG, 'step_' + str(self.step)) # Cleanup old configs generated in previous versions self._cleanup_old_configs() # Make sure config_path exists # Note: it'll cleanup old configs before creating new ones. self._create_dir(config_path) # Generate the container configs config = self._get_config(self._merge_volumes_configs(data)) for k, v in config.items(): config_dest = os.path.join(config_path, k + '.json') self._update_container_config(config_dest, v) # Update container-startup-config with new config hashes self._update_hashes() self.module.exit_json(**self.results) def _merge_volumes_configs(self, data): """Returns a list of puppet configs with unique config_volume keys. :param data: list :returns: list This method takes in input a list of container puppet configs and returns a list of container puppet configs with unique config_volume keys. It will allow to run puppet for a single volume at a time and avoid the situation where multiple configs using the same config volume would run separately; which would cause race condition issues because of the rsync commands executed at the end of puppet run. To also saves time we support configuring 'shared' services at the same time. For example configuring all of the heat services in a single container pass makes sense and will save some time. To support this we merge shared settings together here. We key off of config_volume as this should be the same for a given group of services. We are also now specifying the container in which the services should be configured. This should match in all instances where the volume name is also the same. """ returned_dict = {} for config in data: config_volume = config.get('config_volume') if config_volume is None or config_volume == '': continue puppet_tags = config.get('puppet_tags') step_config = config.get('step_config') config_image = config.get('config_image') volumes = config.get('volumes') if config_volume in returned_dict: # A config already exists for that config_volume, # we'll append puppet_tags and step_config and extend volumes. config_image_orig = ( returned_dict[config_volume]['config_image'] ) if volumes: volumes_orig = returned_dict[config_volume].get('volumes', []) if volumes_orig: volumes = volumes_orig.extend(volumes) returned_dict[config_volume]['volumes'] = ( sorted(set(volumes)) ) if puppet_tags is not None: returned_dict[config_volume]['puppet_tags'] = '%s,%s' % ( returned_dict[config_volume]['puppet_tags'], puppet_tags ) if step_config is not None: returned_dict[config_volume]['step_config'] = '%s\n%s' % ( returned_dict[config_volume]['step_config'], step_config ) if config_image != config_image_orig: self.module.warn('{} config image does not match with ' '{}'.format(config_image, config_image_orig)) else: # This is a new config returned_dict[config_volume] = config return returned_dict def _get_config(self, data): """Returns a list of puppet configs per container. :param data: list :returns: list This method takes in input a list of dicts and returns a dictionary which match with the podman_container module interface. """ returned_dict = {} default_volumes = ['/etc/localtime:/etc/localtime:ro', '/etc/puppet:/tmp/puppet-etc:ro', '/etc/pki/ca-trust/extracted:' '/etc/pki/ca-trust/extracted:ro', '/etc/pki/tls/certs/ca-bundle.crt:' '/etc/pki/tls/certs/ca-bundle.crt:ro', '/etc/pki/tls/certs/ca-bundle.trust.crt:' '/etc/pki/tls/certs/ca-bundle.trust.crt:ro', '/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro', '%s:/var/lib/config-data' ':rw' % self.config_vol_prefix, '/var/lib/container-puppet/puppetlabs/facter.conf:' '/etc/puppetlabs/facter/facter.conf:ro', '/var/lib/container-puppet/puppetlabs:' '/opt/puppetlabs:ro', '%s:%s:ro' % (CONTAINER_ENTRYPOINT, CONTAINER_ENTRYPOINT), '/usr/share/openstack-puppet/modules:' '/usr/share/openstack-puppet/modules:ro', '/dev/log:/dev/log:rw'] # Defaults default_data = { # the security_opt can be removed once we properly address: # https://bugs.launchpad.net/tripleo/+bug/1864501 'security_opt': 'label=disable', 'user': 0, 'entrypoint': CONTAINER_ENTRYPOINT, 'environment': self._get_environment_config() } for config_volume, config in data.items(): cdata = copy.deepcopy(default_data) volumes = copy.deepcopy(default_volumes) cname = 'container-puppet-' + config_volume if self.check: volumes += ['/etc/puppet/check-mode:' '/tmp/puppet-check-mode:ro'] if self.net_host: cdata['net'] = 'host' volumes += ['/etc/hosts:/etc/hosts:ro'] else: cdata['net'] = 'none' cdata['environment']['PUPPET_TAGS'] = ( self._get_puppet_tags(config)) cdata['environment']['NAME'] = config_volume for k, v in config.items(): if k == 'config_volume': continue if k == 'puppet_tags': continue if k == 'step_config': step_config = 'include ::tripleo::packages\n' + v cdata['environment']['STEP_CONFIG'] = step_config continue if k == 'config_image': cdata['image'] = v continue if k == 'privileged': cdata['privileged'] = v continue if k == 'volumes': if isinstance(v, (list)): volumes.extend(v) else: volumes += [v] continue # Keep this one at the very end to override any attribute: cdata[k] = v cdata['volumes'] = sorted(set(volumes)) returned_dict[cname] = cdata return returned_dict def _get_environment_config(self): """Returns common environment configs. :returns: dict """ returned_env = { 'STEP': self._get_puppet_step(self.step), 'NET_HOST': str(self.net_host).lower(), 'DEBUG': str(self.debug).lower(), } if self.hostname is not None: returned_env['HOSTNAME'] = self.hostname if not self.no_archive: returned_env['NO_ARCHIVE'] = '' else: returned_env['NO_ARCHIVE'] = self.no_archive return returned_env def _get_puppet_step(self, step): """Returns the step used by Puppet during a run." :param step: integer :returns: integer """ # When container_puppet_config is called at step1, it's to initialize # configuration files for all services like they were deployed; so # in Puppet it means after step5. Which is why we override the step # just for the Puppet run. # Note that it was the same behavior with container-puppet.py since # STEP was set to 6 by default and wasn't overriden when the script # was run at step1. if step == 1: return 6 return step def _get_puppet_tags(self, config): """Returns Puppet tags. :returns: string """ puppet_tags = 'file,file_line,concat,augeas,cron' config_puppet_tags = config.get('puppet_tags') if config_puppet_tags is not None: puppet_tags += ',%s' % config_puppet_tags return puppet_tags def _exists(self, path): """Returns True if a patch exists. :param path: string :returns: boolean """ if os.path.exists(path): return True def _remove_dir(self, path): """Remove a directory. :param path: string """ if self._exists(path): shutil.rmtree(path) def _remove_file(self, path): """Remove a file. :param path: string """ if self._exists(path): os.remove(path) def _create_dir(self, path): """Creates a directory. :param path: string """ if self._exists(path): self._remove_dir(path) os.makedirs(path) def _find(self, path, pattern='*.json'): """Returns a list of files in a directory. :param path: string :param pattern: string :returns: list """ configs = [] if self._exists(path): for root, dirnames, filenames in os.walk(path): for filename in fnmatch.filter(filenames, pattern): configs.append(os.path.join(root, filename)) else: self.module.warn('{} does not exists'.format(path)) return configs def _slurp(self, path): """Slurps a file and return its content. :param path: string :returns: string """ if self._exists(path): f = open(path, 'r') return f.read() else: self.module.warn('{} was not found.'.format(path)) return '' def _update_container_config(self, path, config): """Update a container config. :param path: string :param config: string """ f = open(path, 'wb') f.write(json.dumps(config, indent=2).encode('utf-8')) os.chmod(path, 0o600) self.results['changed'] = True def _get_config_hash(self, config_volume): """Returns a config hash from a config_volume. :param config_volume: string :returns: string """ hashfile = "%s.md5sum" % config_volume hash_data = '' if self._exists(hashfile): return self._slurp(hashfile).strip('\n') def _get_config_base(self, prefix, volume): """Returns a config base path for a specific volume. :param prefix: string :param volume: string :returns: string """ # crawl the volume's path upwards until we find the # volume's base, where the hashed config file resides path = volume base = prefix.rstrip(os.path.sep) base_generated = os.path.join(base, 'puppet-generated') while path.startswith(prefix): dirname = os.path.dirname(path) if dirname == base or dirname == base_generated: return path else: path = dirname self.module.fail_json( msg='Could not find config base for: {} ' 'with prefix: {}'.format(volume, prefix)) def _match_config_volumes(self, config): """Return a list of volumes that match a config. :param config: dict :returns: list """ # Match the mounted config volumes - we can't just use the # key as e.g "novacomute" consumes config-data/nova prefix = self.config_vol_prefix try: volumes = config.get('volumes', []) except AttributeError: self.module.fail_json( msg='Error fetching volumes. Prefix: ' '{} - Config: {}'.format(prefix, config)) return sorted([self._get_config_base(prefix, v.split(":")[0]) for v in volumes if v.startswith(prefix)]) def _update_hashes(self): """Update container startup config with new config hashes if needed. """ configs = self._find(CONTAINER_STARTUP_CONFIG) for config in configs: old_config_hash = '' cname = os.path.splitext(os.path.basename(config))[0] if cname.startswith('hashed-'): # Take the opportunity to cleanup old hashed files which # don't exist anymore. self._remove_file(config) continue startup_config_json = json.loads(self._slurp(config)) config_volumes = self._match_config_volumes(startup_config_json) config_hashes = [ self._get_config_hash(vol_path) for vol_path in config_volumes ] config_hashes = filter(None, config_hashes) if 'environment' in startup_config_json: old_config_hash = startup_config_json['environment'].get( 'TRIPLEO_CONFIG_HASH', '') if config_hashes is not None and config_hashes: config_hash = '-'.join(config_hashes) if config_hash == old_config_hash: # config doesn't need an update continue self.module.warn('Config change detected for {}, new ' 'hash: {}'.format(cname, config_hash)) if 'environment' not in startup_config_json: startup_config_json['environment'] = {} startup_config_json['environment']['TRIPLEO_CONFIG_HASH'] = ( config_hash) self._update_container_config(config, startup_config_json) def _cleanup_old_configs(self): """Cleanup old container configurations and directories. """ # This configuration file was removed here: # https://review.opendev.org/#/c/702876 old_config = os.path.join(CONTAINER_STARTUP_CONFIG + '-step_' + str(self.step) + '.json') self._remove_file(old_config) def main(): module = AnsibleModule( argument_spec=yaml.safe_load(DOCUMENTATION)['options'], supports_check_mode=True, ) results = dict( changed=False ) ContainerPuppetManager(module, results) if __name__ == '__main__': main()