From b5a33168a7289398e499ca7f10ac2ea7680fb6ea Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Tue, 15 Oct 2019 14:30:58 -0400 Subject: [PATCH] Generate startup configs files per step and per container For each step and each container, generate a unique JSON, and Paunch will be able to read them all thanks to the patch in dependency (tripleo-ansible). Old location: /var/lib/tripleo-config/container-startup-config-step1.json We keep the old files for backward compatibility. New location: /var/lib/tripleo-config/container-startup-config/step_1/haproxy.json Note: hashed files won't be generated for the old location anymore, since it's done via container-puppet.py in which we now give the new location. Story: 2006732 Task: 37162 Depends-On: If0f1c6c308cd58f7baa9a8449fbf685ff10f0e0a Change-Id: I1cf8923a698d0f6e0b1e00a7985f363a83e914c4 --- common/container-puppet.py | 31 +++++++++++++-------- common/container_startup_configs_tasks.yaml | 18 ++++++++++++ common/deploy-steps-tasks-step-1.yaml | 25 ++++++++++++++++- common/deploy-steps-tasks.yaml | 4 +-- common/deploy-steps.j2 | 1 + 5 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 common/container_startup_configs_tasks.yaml diff --git a/common/container-puppet.py b/common/container-puppet.py index 1b9ec81fdf..d1741eb241 100755 --- a/common/container-puppet.py +++ b/common/container-puppet.py @@ -461,10 +461,17 @@ for returncode, config_volume in zip(returncodes, config_volumes): # Update the startup configs with the config hash we generated above -startup_configs = os.environ.get('STARTUP_CONFIG_PATTERN', '/var/lib/tripleo-config/docker-container-startup-config-step_*.json') +startup_configs = os.environ.get('STARTUP_CONFIG_PATTERN', '/var/lib/tripleo-config/container_startup_config/*/*.json') log.debug('STARTUP_CONFIG_PATTERN: %s' % startup_configs) infiles = glob.glob(startup_configs) + for infile in infiles: + # If the JSON is already hashed, we'll skip it; and a new hashed file will + # be created if config changed. + if 'hashed' in infile: + log.debug('%s skipped, already hashed' % infile) + continue + with open(infile) as f: infile_data = json.load(f) @@ -473,17 +480,17 @@ for infile in infiles: if not infile_data: infile_data = {} - for k, v in iter(infile_data.items()): - config_volumes = match_config_volumes(config_volume_prefix, v) - config_hashes = [get_config_hash(volume_path) for volume_path in config_volumes] - config_hashes = filter(None, config_hashes) - config_hash = '-'.join(config_hashes) - if config_hash: - log.debug("Updating config hash for %s, config_volume=%s hash=%s" % (k, config_volume, config_hash)) - # When python 27 support is removed, we will be able to use z = {**x, **y} to merge the dicts. - v.get('environment', {}).update({'TRIPLEO_CONFIG_HASH': config_hash}) - env = v.get('environment') - infile_data[k]['environment'] = env + c_name = os.path.splitext(os.path.basename(infile))[0] + config_volumes = match_config_volumes(config_volume_prefix, infile_data) + config_hashes = [get_config_hash(volume_path) for volume_path in config_volumes] + config_hashes = filter(None, config_hashes) + config_hash = '-'.join(config_hashes) + if config_hash: + log.debug("Updating config hash for %s, config_volume=%s hash=%s" % (c_name, config_volume, config_hash)) + # When python 27 support is removed, we will be able to use z = {**x, **y} to merge the dicts. + infile_data.get('environment', {}).update({'TRIPLEO_CONFIG_HASH': config_hash}) + env = infile_data.get('environment') + infile_data['environment'] = env outfile = os.path.join(os.path.dirname(infile), "hashed-" + os.path.basename(infile)) with open(outfile, 'w') as out_f: diff --git a/common/container_startup_configs_tasks.yaml b/common/container_startup_configs_tasks.yaml new file mode 100644 index 0000000000..a3b1d1403e --- /dev/null +++ b/common/container_startup_configs_tasks.yaml @@ -0,0 +1,18 @@ +--- +- set_fact: + step_path: "{{ item.0 }}" + +- name: "Create directory for {{ step_path }} container startup configs" + file: + path: "/var/lib/tripleo-config/container-startup-config/{{ step_path }}/" + mode: 0600 + recurse: yes + setype: svirt_sandbox_file_t + +- name: "Creating container startup configs for {{ step_path }}" + copy: + content: "{{ item.value | to_nice_json }}" + dest: "/var/lib/tripleo-config/container-startup-config/{{ step_path }}/{{ item.key }}.json" + setype: svirt_sandbox_file_t + mode: 0600 + loop: "{{ item.1 | dict2items }}" diff --git a/common/deploy-steps-tasks-step-1.yaml b/common/deploy-steps-tasks-step-1.yaml index 5a6415e777..d83d1a5455 100644 --- a/common/deploy-steps-tasks-step-1.yaml +++ b/common/deploy-steps-tasks-step-1.yaml @@ -247,7 +247,23 @@ tags: - container_startup_configs - - name: Write per-step container startup configs + # This file location is deprecated and the new location is now: + # /var/lib/tripleo-config/container-startup-config/step_X/.json + # Can be removed in V cycle + - name: Write /var/lib/tripleo-config/container-startup-config-readme.txt + no_log: True + copy: + content: "Container startup configs moved to /var/lib/tripleo-config/container-startup-config" + dest: /var/lib/tripleo-config/container-startup-config-readme.txt + force: yes + mode: '0600' + tags: + - container_startup_configs + + # For backward compatibility, the files are still generated. + # However, container_puppet.py won't create their hashed version if + # config changed. + - name: Write per-step container startup configs for backward compatibility no_log: True copy: content: "{{ item[1] | to_nice_json }}" @@ -260,6 +276,13 @@ tags: - container_startup_configs + - name: Generate startup configs files per step and per container + no_log: True + include_tasks: container_startup_configs_tasks.yaml + loop: "{{ container_config_with_default | dictsort }}" + tags: + - container_startup_configs + - name: Create /var/lib/kolla/config_files directory file: path: /var/lib/kolla/config_files diff --git a/common/deploy-steps-tasks.yaml b/common/deploy-steps-tasks.yaml index 96d689a573..e52198c974 100644 --- a/common/deploy-steps-tasks.yaml +++ b/common/deploy-steps-tasks.yaml @@ -174,7 +174,7 @@ CONFIG: '/var/lib/container-puppet/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}container-puppet.json' CONFIG_VOLUME_PREFIX: '/var/lib/config-data{{ ansible_check_mode | bool | ternary("/check-mode", "") }}' CHECK_MODE: '{{ ansible_check_mode | bool | ternary(1, 0) }}' - STARTUP_CONFIG_PATTERN: '/var/lib/tripleo-config/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}container-startup-config-step_*.json' + STARTUP_CONFIG_PATTERN: '/var/lib/tripleo-config/container-startup-config/*/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}*.json' MOUNT_HOST_PUPPET: '{{docker_puppet_mount_host_puppet | default(true)}}' CONTAINER_LOG_STDOUT_PATH: "{{ container_log_stdout_path }}" CONTAINER_HEALTHCHECK_DISABLED: "{{ container_healthcheck_disabled }}" @@ -233,7 +233,7 @@ environment: TRIPLEO_MINOR_UPDATE: '{{ tripleo_minor_update | default(false) }}' paunch: - config: "/var/lib/tripleo-config/hashed-container-startup-config-step_{{ step }}.json" + config: "/var/lib/tripleo-config/container-startup-config/step_{{ step }}" config_id: "tripleo_step{{ step }}" action: apply container_cli: "{{ container_cli }}" diff --git a/common/deploy-steps.j2 b/common/deploy-steps.j2 index c43704ab7a..b74fa737a3 100644 --- a/common/deploy-steps.j2 +++ b/common/deploy-steps.j2 @@ -373,6 +373,7 @@ outputs: all_nodes_extra_map_data: {get_param: AllNodesExtraMapData} common_deploy_steps_tasks: {get_file: deploy-steps-tasks.yaml} common_deploy_steps_tasks_step_1: {get_file: deploy-steps-tasks-step-1.yaml} + container_startup_configs_tasks: {get_file: container_startup_configs_tasks.yaml} docker_puppet_script: {get_file: ./container-puppet.py} container_puppet_script: {get_file: ./container-puppet.sh} all_nodes_validation_script.sh : {get_file: ../validation-scripts/all-nodes.sh}