Generate startup configs files per step and per container

For each step and each container, generate a unique JSON, and Paunch
will be able to read them all thanks to the patch in dependency
(tripleo-ansible).

Old location:
/var/lib/tripleo-config/container-startup-config-step1.json
We keep the old files for backward compatibility.

New location:
/var/lib/tripleo-config/container-startup-config/step_1/haproxy.json

Note: hashed files won't be generated for the old location anymore,
since it's done via container-puppet.py in which we now give the new
location.

Story: 2006732
Task: 37162
Depends-On: If0f1c6c308cd58f7baa9a8449fbf685ff10f0e0a
Change-Id: I1cf8923a698d0f6e0b1e00a7985f363a83e914c4

common/container-puppet.py

@@ -461,10 +461,17 @@ for returncode, config_volume in zip(returncodes, config_volumes):
 
 
 # Update the startup configs with the config hash we generated above
-startup_configs = os.environ.get('STARTUP_CONFIG_PATTERN', '/var/lib/tripleo-config/docker-container-startup-config-step_*.json')
+startup_configs = os.environ.get('STARTUP_CONFIG_PATTERN', '/var/lib/tripleo-config/container_startup_config/*/*.json')
 log.debug('STARTUP_CONFIG_PATTERN: %s' % startup_configs)
 infiles = glob.glob(startup_configs)
+
 for infile in infiles:
+    # If the JSON is already hashed, we'll skip it; and a new hashed file will
+    # be created if config changed.
+    if 'hashed' in infile:
+        log.debug('%s skipped, already hashed' % infile)
+        continue
+
     with open(infile) as f:
         infile_data = json.load(f)
 
@@ -473,17 +480,17 @@ for infile in infiles:
     if not infile_data:
         infile_data = {}
 
-    for k, v in iter(infile_data.items()):
-        config_volumes = match_config_volumes(config_volume_prefix, v)
-        config_hashes = [get_config_hash(volume_path) for volume_path in config_volumes]
-        config_hashes = filter(None, config_hashes)
-        config_hash = '-'.join(config_hashes)
-        if config_hash:
-            log.debug("Updating config hash for %s, config_volume=%s hash=%s" % (k, config_volume, config_hash))
-            # When python 27 support is removed, we will be able to use z = {**x, **y} to merge the dicts.
-            v.get('environment', {}).update({'TRIPLEO_CONFIG_HASH': config_hash})
-            env = v.get('environment')
-            infile_data[k]['environment'] = env
+    c_name = os.path.splitext(os.path.basename(infile))[0]
+    config_volumes = match_config_volumes(config_volume_prefix, infile_data)
+    config_hashes = [get_config_hash(volume_path) for volume_path in config_volumes]
+    config_hashes = filter(None, config_hashes)
+    config_hash = '-'.join(config_hashes)
+    if config_hash:
+        log.debug("Updating config hash for %s, config_volume=%s hash=%s" % (c_name, config_volume, config_hash))
+        # When python 27 support is removed, we will be able to use z = {**x, **y} to merge the dicts.
+        infile_data.get('environment', {}).update({'TRIPLEO_CONFIG_HASH': config_hash})
+        env = infile_data.get('environment')
+        infile_data['environment'] = env
 
     outfile = os.path.join(os.path.dirname(infile), "hashed-" + os.path.basename(infile))
     with open(outfile, 'w') as out_f:

common/container_startup_configs_tasks.yaml

@@ -0,0 +1,18 @@
+---
+- set_fact:
+    step_path: "{{ item.0 }}"
+
+- name: "Create directory for {{ step_path }} container startup configs"
+  file:
+    path: "/var/lib/tripleo-config/container-startup-config/{{ step_path }}/"
+    mode: 0600
+    recurse: yes
+    setype: svirt_sandbox_file_t
+
+- name: "Creating container startup configs for {{ step_path }}"
+  copy:
+    content: "{{ item.value | to_nice_json }}"
+    dest: "/var/lib/tripleo-config/container-startup-config/{{ step_path }}/{{ item.key }}.json"
+    setype: svirt_sandbox_file_t
+    mode: 0600
+  loop: "{{ item.1 | dict2items }}"

common/deploy-steps-tasks-step-1.yaml

@@ -247,7 +247,23 @@
           tags:
             - container_startup_configs
 
-        - name: Write per-step container startup configs
+        # This file location is deprecated and the new location is now:
+        # /var/lib/tripleo-config/container-startup-config/step_X/<container_name>.json
+        # Can be removed in V cycle
+        - name: Write /var/lib/tripleo-config/container-startup-config-readme.txt
+          no_log: True
+          copy:
+            content: "Container startup configs moved to /var/lib/tripleo-config/container-startup-config"
+            dest: /var/lib/tripleo-config/container-startup-config-readme.txt
+            force: yes
+            mode: '0600'
+          tags:
+            - container_startup_configs
+
+        # For backward compatibility, the files are still generated.
+        # However, container_puppet.py won't create their hashed version if
+        # config changed.
+        - name: Write per-step container startup configs for backward compatibility
           no_log: True
           copy:
             content: "{{ item[1] | to_nice_json }}"
@@ -260,6 +276,13 @@
           tags:
             - container_startup_configs
 
+        - name: Generate startup configs files per step and per container
+          no_log: True
+          include_tasks: container_startup_configs_tasks.yaml
+          loop: "{{ container_config_with_default | dictsort }}"
+          tags:
+            - container_startup_configs
+
         - name: Create /var/lib/kolla/config_files directory
           file:
             path: /var/lib/kolla/config_files

common/deploy-steps-tasks.yaml

@@ -174,7 +174,7 @@
         CONFIG: '/var/lib/container-puppet/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}container-puppet.json'
         CONFIG_VOLUME_PREFIX: '/var/lib/config-data{{ ansible_check_mode | bool | ternary("/check-mode", "") }}'
         CHECK_MODE: '{{ ansible_check_mode | bool | ternary(1, 0) }}'
-        STARTUP_CONFIG_PATTERN: '/var/lib/tripleo-config/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}container-startup-config-step_*.json'
+        STARTUP_CONFIG_PATTERN: '/var/lib/tripleo-config/container-startup-config/*/{{ ansible_check_mode | bool | ternary("check-mode/", "") }}*.json'
         MOUNT_HOST_PUPPET: '{{docker_puppet_mount_host_puppet | default(true)}}'
         CONTAINER_LOG_STDOUT_PATH: "{{ container_log_stdout_path }}"
         CONTAINER_HEALTHCHECK_DISABLED: "{{ container_healthcheck_disabled }}"
@@ -233,7 +233,7 @@
       environment:
         TRIPLEO_MINOR_UPDATE: '{{ tripleo_minor_update | default(false) }}'
       paunch:
-        config: "/var/lib/tripleo-config/hashed-container-startup-config-step_{{ step }}.json"
+        config: "/var/lib/tripleo-config/container-startup-config/step_{{ step }}"
         config_id: "tripleo_step{{ step }}"
         action: apply
         container_cli: "{{ container_cli }}"

common/deploy-steps.j2

@@ -373,6 +373,7 @@ outputs:
         all_nodes_extra_map_data: {get_param: AllNodesExtraMapData}
       common_deploy_steps_tasks: {get_file: deploy-steps-tasks.yaml}
       common_deploy_steps_tasks_step_1: {get_file: deploy-steps-tasks-step-1.yaml}
+      container_startup_configs_tasks: {get_file: container_startup_configs_tasks.yaml}
       docker_puppet_script: {get_file: ./container-puppet.py}
       container_puppet_script: {get_file: ./container-puppet.sh}
       all_nodes_validation_script.sh : {get_file: ../validation-scripts/all-nodes.sh}