From 80d12514d5cd3c20057bd01588e5d5d15d131ca9 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobreli@redhat.com>
Date: Thu, 4 Jul 2019 15:27:23 +0200
Subject: [PATCH] Fix generating Apache configs by container-puppet

The changes listed below provide a single unit of work required to
configure Apache backend for WSGI-based OpenStack API services
w/o conflicts causing containers startup failures.

W/o this change /etc/httpd/conf.modules.d/00-mpm.conf shipped with RPM
or other conflicting httpd modules might remain in the containers
and cause startup failures. While puppet removes such conflicts from
the configuration, f.e. when switching MPM 'prefork' to 'event', and we
expect it never gets into container configs.

Make kolla extended start properly enforcing the wanted state of
/etc/httpd, including conf.d and conf.modules.d, and also any of the
removed by puppet files, like conflicting Apache MPM modules.

Add container-puppet tasks to ensure apache MPM configs generated
before the main config steps that require Apache started in the
service container.

Additionally, ensure consistent mirroring across config-data
paths for the container-puppet tool. Purge obsoleted/irrelevant files
in the destingation (puppet-generated) before rsyncing new contents
into it.

Closes-Bug: #1835414

Change-Id: I3e5b4372a01b29bf13179d8a16acc36da9c5caab
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
---
 common/container-puppet.py                    | 36 ++++++++++++-------
 .../aodh/aodh-api-container-puppet.yaml       | 13 ++++++-
 .../barbican-api-container-puppet.yaml        | 13 ++++++-
 .../cinder/cinder-api-container-puppet.yaml   | 12 ++++++-
 .../panko/panko-api-container-puppet.yaml     | 13 ++++++-
 deployment/ec2/ec2-api-container-puppet.yaml  | 15 ++++++--
 .../glance/glance-api-container-puppet.yaml   | 13 ++++++-
 .../gnocchi/gnocchi-api-container-puppet.yaml | 13 ++++++-
 .../heat/heat-api-cfn-container-puppet.yaml   | 13 ++++++-
 .../heat/heat-api-container-puppet.yaml       | 13 ++++++-
 .../horizon/horizon-container-puppet.yaml     | 13 ++++++-
 .../ironic/ironic-api-container-puppet.yaml   | 13 ++++++-
 .../ironic/ironic-pxe-container-puppet.yaml   | 13 ++++++-
 .../keystone/keystone-container-puppet.yaml   | 12 ++++++-
 .../manila/manila-api-container-puppet.yaml   | 13 ++++++-
 .../neutron/neutron-api-container-puppet.yaml | 13 ++++++-
 .../nova/nova-api-container-puppet.yaml       | 13 ++++++-
 .../nova/nova-metadata-container-puppet.yaml  | 13 ++++++-
 .../octavia/octavia-api-container-puppet.yaml | 13 ++++++-
 .../placement-api-container-puppet.yaml       | 17 ++++++++-
 .../swift/swift-proxy-container-puppet.yaml   | 13 ++++++-
 deployment/zaqar/zaqar-container-puppet.yaml  | 13 ++++++-
 22 files changed, 279 insertions(+), 34 deletions(-)

diff --git a/common/container-puppet.py b/common/container-puppet.py
index 8b164667e5..b240344ff6 100755
--- a/common/container-puppet.py
+++ b/common/container-puppet.py
@@ -289,7 +289,8 @@ if not os.path.exists(sh_script):
         # Create a reference timestamp to easily find all files touched by
         # puppet. The sync ensures we get all the files we want due to
         # different timestamp.
-        origin_of_time=/var/lib/config-data/${NAME}.origin_of_time
+        conf_data_path="/var/lib/config-data/${NAME}"
+        origin_of_time="${conf_data_path}.origin_of_time"
         touch $origin_of_time
         sync
 
@@ -343,7 +344,7 @@ if not os.path.exists(sh_script):
 
             exclude_files=""
             for p in $password_files; do
-                if [ -f "$p" -a -f "/var/lib/config-data/${NAME}$p" ]; then
+                if [ -f "$p" -a -f "${conf_data_path}$p" ]; then
                     exclude_files+=" --exclude=$p"
                 fi
             done
@@ -357,23 +358,33 @@ if not os.path.exists(sh_script):
                 fi
             done
 
-            echo "Rsyncing config files from ${rsync_srcs} into /var/lib/config-data/${NAME}"
-            rsync -a $verbosity -R --delay-updates --delete-after $exclude_files $rsync_srcs /var/lib/config-data/${NAME}
+            echo "Evaluating config files to be removed for the $NAME configuration"
+            rsync -av -R --dry-run --delete-after $exclude_files $rsync_srcs ${conf_data_path} |\
+                awk '/^deleting/ {print $2}' > /tmp/files_to_purge
+
+            echo "Rsyncing config files from ${rsync_srcs} into ${conf_data_path}"
+            rsync -a $verbosity -R --delay-updates --delete-after $exclude_files $rsync_srcs ${conf_data_path}
 
 
             # Also make a copy of files modified during puppet run
-            # This is useful for debugging
             echo "Gathering files modified after $(stat -c '%y' $origin_of_time)"
-            mkdir -p /var/lib/config-data/puppet-generated/${NAME}
+
+            # Purge obsoleted contents to maintain a fresh and filtered mirror
+            puppet_generated_path=/var/lib/config-data/puppet-generated/${NAME}
+            mkdir -p ${puppet_generated_path}
+            echo "Ensuring the removed config files are also purged in ${puppet_generated_path}:"
+            cat /tmp/files_to_purge | sort
+            cat /tmp/files_to_purge | xargs -n1 -r -I{} \
+                bash -c "test -f ${puppet_generated_path}/{} && rm -f ${puppet_generated_path}/{}"
             exec 5>&1
             exec 1>/tmp/files_from
             find $rsync_srcs -newer $origin_of_time -not -path '/etc/puppet*' -print0
             exec 1>&5
             echo "Files modified during puppet run:"
             cat /tmp/files_from | sort | xargs -0 printf "%s\n"
-            echo "Rsyncing the modified files into /var/lib/config-data/puppet-generated/${NAME}"
+            echo "Rsyncing the modified files into ${puppet_generated_path}"
             rsync -a $verbosity -R -0 --delay-updates --delete-after $exclude_files \
-                --files-from=/tmp/files_from / /var/lib/config-data/puppet-generated/${NAME}
+                --files-from=/tmp/files_from / ${puppet_generated_path}
 
             # Write a checksum of the config-data dir, this is used as a
             # salt to trigger container restart when the config changes
@@ -395,10 +406,10 @@ if not os.path.exists(sh_script):
             # output because otherwise the sed command cannot work. The sed is
             # needed because puppet puts timestamps as comments in cron and
             # parsedfile resources, hence triggering a change at every redeploy
-            tar -c --mtime='1970-01-01' $EXCLUDE -f - /var/lib/config-data/${NAME} $additional_checksum_files | tar xO | \
-                    sed '/^#.*HEADER.*/d' | md5sum | awk '{print $1}' > /var/lib/config-data/${NAME}.md5sum
-            tar -c --mtime='1970-01-01' $EXCLUDE -f - /var/lib/config-data/puppet-generated/${NAME} $additional_checksum_files --mtime='1970-01-01' | tar xO \
-                    | sed '/^#.*HEADER.*/d' | md5sum | awk '{print $1}' > /var/lib/config-data/puppet-generated/${NAME}.md5sum
+            tar -c --mtime='1970-01-01' $EXCLUDE -f - ${conf_data_path} $additional_checksum_files | tar xO | \
+                    sed '/^#.*HEADER.*/d' | md5sum | awk '{print $1}' > ${conf_data_path}.md5sum
+            tar -c --mtime='1970-01-01' $EXCLUDE -f - ${puppet_generated_path} $additional_checksum_files --mtime='1970-01-01' | tar xO \
+                    | sed '/^#.*HEADER.*/d' | md5sum | awk '{print $1}' > ${puppet_generated_path}.md5sum
         fi
         """)
 
@@ -420,6 +431,7 @@ def mp_puppet_config(*args):
     with tempfile.NamedTemporaryFile() as tmp_man:
         with open(tmp_man.name, 'w') as man_file:
             man_file.write('include ::tripleo::packages\n')
+            man_file.write('noop_resource("service")\n')
             man_file.write(manifest)
 
         uname = RUNNER.unique_container_name('container-puppet-%s' %
diff --git a/deployment/aodh/aodh-api-container-puppet.yaml b/deployment/aodh/aodh-api-container-puppet.yaml
index 55f4a89373..b0f644a952 100644
--- a/deployment/aodh/aodh-api-container-puppet.yaml
+++ b/deployment/aodh/aodh-api-container-puppet.yaml
@@ -142,7 +142,8 @@ outputs:
             - "\n"
             - - "include tripleo::profile::base::aodh::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerAodhConfigImage}
+        config_image: &aodh_config_image
+          {get_param: ContainerAodhConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/aodh_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -151,6 +152,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -214,6 +219,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'aodh'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *aodh_config_image
       host_prep_tasks:
       - name: create persistent logs directory
         file:
diff --git a/deployment/barbican/barbican-api-container-puppet.yaml b/deployment/barbican/barbican-api-container-puppet.yaml
index f8bb645588..39f4c218c7 100644
--- a/deployment/barbican/barbican-api-container-puppet.yaml
+++ b/deployment/barbican/barbican-api-container-puppet.yaml
@@ -281,7 +281,8 @@ outputs:
             - "\n"
             - - "include ::tripleo::profile::base::barbican::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerBarbicanConfigImage}
+        config_image: &barbican_config_image
+          {get_param: ContainerBarbicanConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/barbican_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -290,6 +291,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -724,6 +729,12 @@ outputs:
                           - /usr/lib64/libnethsm.so:/usr/lib64/libnethsm.so
                         - null
                 environment: *kolla_env
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'barbican'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *barbican_config_image
       host_prep_tasks:
         list_concat:
           - {get_attr: [BarbicanApiLogging, host_prep_tasks]}
diff --git a/deployment/cinder/cinder-api-container-puppet.yaml b/deployment/cinder/cinder-api-container-puppet.yaml
index 0f0f22826f..28ae049c58 100644
--- a/deployment/cinder/cinder-api-container-puppet.yaml
+++ b/deployment/cinder/cinder-api-container-puppet.yaml
@@ -217,7 +217,8 @@ outputs:
             - "\n"
             - - "include ::tripleo::profile::base::cinder::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerCinderConfigImage}
+        config_image: &cinder_config_image
+            {get_param: ContainerCinderConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/cinder_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -226,6 +227,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -325,6 +330,11 @@ outputs:
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       container_puppet_tasks:
+        step_3:
+          config_volume: 'cinder'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *cinder_config_image
         # Create the default volume type after the cinder-api service is
         # running. The puppet code will ensure this is only done once, on
         # the bootstrap node.
diff --git a/deployment/deprecated/panko/panko-api-container-puppet.yaml b/deployment/deprecated/panko/panko-api-container-puppet.yaml
index c060e98013..87e0502a4a 100644
--- a/deployment/deprecated/panko/panko-api-container-puppet.yaml
+++ b/deployment/deprecated/panko/panko-api-container-puppet.yaml
@@ -229,7 +229,8 @@ outputs:
             - "\n"
             - - "include tripleo::profile::base::panko::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerPankoConfigImage}
+        config_image: &panko_config_image
+          {get_param: ContainerPankoConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/panko_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -238,6 +239,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -327,6 +332,12 @@ outputs:
                   - /var/lib/config-data/puppet-generated/panko/:/var/lib/kolla/config_files/src:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'panko'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *panko_config_image
       host_prep_tasks: {get_attr: [PankoApiLogging, host_prep_tasks]}
       metadata_settings:
         get_attr: [ApacheServiceBase, role_data, metadata_settings]
diff --git a/deployment/ec2/ec2-api-container-puppet.yaml b/deployment/ec2/ec2-api-container-puppet.yaml
index ff9acf32ec..caf3dbd725 100644
--- a/deployment/ec2/ec2-api-container-puppet.yaml
+++ b/deployment/ec2/ec2-api-container-puppet.yaml
@@ -226,7 +226,8 @@ outputs:
             - "\n"
             - - "include tripleo::profile::base::nova::ec2api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerEc2ApiConfigImage}
+        config_image: &ec2_api_config_image
+          {get_param: ContainerEc2ApiConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/ec2_api.json:
           command: /usr/bin/ec2-api
@@ -246,6 +247,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -338,6 +343,12 @@ outputs:
                     environment:
                       - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
                 - {}
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'ec2_api'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *ec2_api_config_image
       host_prep_tasks:
         - name: create persistent log directories
           file:
@@ -348,7 +359,7 @@ outputs:
             - { 'path': /var/log/containers/ec2_api, 'setype': svirt_sandbox_file_t }
             - { 'path': /var/log/containers/ec2_api_metadata, 'setype': svirt_sandbox_file_t }
             - { 'path': /var/log/ec2_api, 'setype': svirt_sandbox_file_t }
-            - { 'path': /var/log/ec2_api_metadata, 'setype': svirt_sandbox_file_t }            
+            - { 'path': /var/log/ec2_api_metadata, 'setype': svirt_sandbox_file_t }
         - name: ec2_api logs readme
           copy:
             dest: /var/log/{{ item }}/readme.txt
diff --git a/deployment/glance/glance-api-container-puppet.yaml b/deployment/glance/glance-api-container-puppet.yaml
index d7784da9eb..03b381832a 100644
--- a/deployment/glance/glance-api-container-puppet.yaml
+++ b/deployment/glance/glance-api-container-puppet.yaml
@@ -462,7 +462,8 @@ outputs:
                   include ::glance::cache::pruner
                 - ''
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerGlanceApiConfigImage}
+        config_image: &glance_config_image
+          {get_param: ContainerGlanceApiConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/glance_api.json:
           command: /usr/bin/glance-api --config-file /usr/share/glance/glance-api-dist.conf --config-file /etc/glance/glance-api.conf --config-file /etc/glance/glance-image-import.conf
@@ -494,6 +495,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -566,6 +571,12 @@ outputs:
                     environment:
                       - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
                 - {}
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'glance_api'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *glance_config_image
       host_prep_tasks:
         list_concat:
         - {get_attr: [GlanceLogging, host_prep_tasks]}
diff --git a/deployment/gnocchi/gnocchi-api-container-puppet.yaml b/deployment/gnocchi/gnocchi-api-container-puppet.yaml
index 3eae7c8a74..1e3d2dfa87 100644
--- a/deployment/gnocchi/gnocchi-api-container-puppet.yaml
+++ b/deployment/gnocchi/gnocchi-api-container-puppet.yaml
@@ -225,7 +225,8 @@ outputs:
         puppet_tags: gnocchi_api_paste_ini,gnocchi_config
         step_config: |
           include ::tripleo::profile::base::gnocchi::api
-        config_image: {get_param: ContainerGnocchiConfigImage}
+        config_image: &gnocchi_config_image
+          {get_param: ContainerGnocchiConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/gnocchi_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -234,6 +235,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -349,6 +354,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_5:
+          config_volume: 'gnocchi'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *gnocchi_config_image
       host_prep_tasks:
         - name: create persistent data and logs directory
           file:
diff --git a/deployment/heat/heat-api-cfn-container-puppet.yaml b/deployment/heat/heat-api-cfn-container-puppet.yaml
index a883c9ee93..6353e5abc4 100644
--- a/deployment/heat/heat-api-cfn-container-puppet.yaml
+++ b/deployment/heat/heat-api-cfn-container-puppet.yaml
@@ -162,7 +162,8 @@ outputs:
         puppet_tags: heat_config,file,concat,file_line
         step_config: |
           include ::tripleo::profile::base::heat::api_cfn
-        config_image: {get_param: ContainerHeatApiCfnConfigImage}
+        config_image: &heat_api_cfn_config_image
+          {get_param: ContainerHeatApiCfnConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/heat_api_cfn.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -171,6 +172,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -212,6 +217,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_4:
+          config_volume: 'heat_api_cfn'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *heat_api_cfn_config_image
       host_prep_tasks: {get_attr: [HeatApiCfnLogging, host_prep_tasks]}
       upgrade_tasks: []
       metadata_settings:
diff --git a/deployment/heat/heat-api-container-puppet.yaml b/deployment/heat/heat-api-container-puppet.yaml
index 56e8daae3b..20b16e703f 100644
--- a/deployment/heat/heat-api-container-puppet.yaml
+++ b/deployment/heat/heat-api-container-puppet.yaml
@@ -178,7 +178,8 @@ outputs:
         puppet_tags: heat_config,file,concat,file_line
         step_config: |
           include ::tripleo::profile::base::heat::api
-        config_image: {get_param: ContainerHeatApiConfigImage}
+        config_image: &heat_api_config_image
+          {get_param: ContainerHeatApiConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/heat_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -187,6 +188,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -260,6 +265,12 @@ outputs:
                   - /var/lib/config-data/puppet-generated/heat_api/:/var/lib/kolla/config_files/src:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_4:
+          config_volume: 'heat_api'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *heat_api_config_image
       host_prep_tasks: {get_attr: [HeatApiLogging, host_prep_tasks]}
       upgrade_tasks: []
       metadata_settings:
diff --git a/deployment/horizon/horizon-container-puppet.yaml b/deployment/horizon/horizon-container-puppet.yaml
index 4ffde6f5eb..59060dd349 100644
--- a/deployment/horizon/horizon-container-puppet.yaml
+++ b/deployment/horizon/horizon-container-puppet.yaml
@@ -211,7 +211,8 @@ outputs:
         puppet_tags: horizon_config
         step_config: |
           include ::tripleo::profile::base::horizon
-        config_image: {get_param: ContainerHorizonConfigImage}
+        config_image: &horizon_config_image
+           {get_param: ContainerHorizonConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/horizon.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -220,6 +221,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -310,6 +315,12 @@ outputs:
               - ENABLE_WATCHER=no
               - ENABLE_ZAQAR=no
               - ENABLE_ZUN=no
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'horizon'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *horizon_config_image
       host_prep_tasks:
         - name: create persistent directories
           file:
diff --git a/deployment/ironic/ironic-api-container-puppet.yaml b/deployment/ironic/ironic-api-container-puppet.yaml
index a9d035c8c0..b5347785c9 100644
--- a/deployment/ironic/ironic-api-container-puppet.yaml
+++ b/deployment/ironic/ironic-api-container-puppet.yaml
@@ -185,7 +185,8 @@ outputs:
             - "\n"
             - - include ::tripleo::profile::base::ironic::api
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerIronicApiConfigImage}
+        config_image: &ironic_api_config_image
+          {get_param: ContainerIronicApiConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/ironic_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -194,6 +195,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -260,6 +265,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'ironic_api'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *ironic_api_config_image
       host_prep_tasks:
         - name: create persistent directories
           file:
diff --git a/deployment/ironic/ironic-pxe-container-puppet.yaml b/deployment/ironic/ironic-pxe-container-puppet.yaml
index b4b3752c30..33344fef44 100644
--- a/deployment/ironic/ironic-pxe-container-puppet.yaml
+++ b/deployment/ironic/ironic-pxe-container-puppet.yaml
@@ -54,7 +54,8 @@ outputs:
         config_volume: ironic
         puppet_tags: ironic_config
         step_config: ''
-        config_image: {get_param: ContainerIronicConfigImage}
+        config_image: &ironic_config_image
+          {get_param: ContainerIronicConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/ironic_pxe_http.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -63,6 +64,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -146,6 +151,12 @@ outputs:
                   - /var/log/containers/httpd/ironic-pxe:/var/log/httpd:z
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_4:
+          config_volume: 'ironic'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *ironic_config_image
       host_prep_tasks:
         - name: create persistent directories
           file:
diff --git a/deployment/keystone/keystone-container-puppet.yaml b/deployment/keystone/keystone-container-puppet.yaml
index 5b63fb0266..587774ee08 100644
--- a/deployment/keystone/keystone-container-puppet.yaml
+++ b/deployment/keystone/keystone-container-puppet.yaml
@@ -666,7 +666,8 @@ outputs:
               - |
                 include ::tripleo::profile::base::keystone
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: &keystone_config_image {get_param: ContainerKeystoneConfigImage}
+        config_image: &keystone_config_image
+            {get_param: ContainerKeystoneConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/keystone.json:
           command: /usr/sbin/httpd
@@ -679,6 +680,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -779,6 +784,11 @@ outputs:
             command:
               [ 'keystone', 'pkill', '--signal', 'USR1', 'httpd' ]
       container_puppet_tasks:
+        step_2:
+          config_volume: 'keystone'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *keystone_config_image
         # Keystone endpoint creation occurs only on single node
         step_3:
           config_volume: 'keystone_init_tasks'
diff --git a/deployment/manila/manila-api-container-puppet.yaml b/deployment/manila/manila-api-container-puppet.yaml
index b3b174159a..6caa7fdec9 100644
--- a/deployment/manila/manila-api-container-puppet.yaml
+++ b/deployment/manila/manila-api-container-puppet.yaml
@@ -164,7 +164,8 @@ outputs:
             - "\n"
             - - "include ::tripleo::profile::base::manila::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerManilaConfigImage}
+        config_image: &manila_config_image
+            {get_param: ContainerManilaConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/manila_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -173,6 +174,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -234,6 +239,12 @@ outputs:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       metadata_settings:
         get_attr: [ApacheServiceBase, role_data, metadata_settings]
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'manila'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *manila_config_image
       host_prep_tasks:
         - name: Create persistent directories
           file:
diff --git a/deployment/neutron/neutron-api-container-puppet.yaml b/deployment/neutron/neutron-api-container-puppet.yaml
index 330f4344d3..21e9da42b9 100644
--- a/deployment/neutron/neutron-api-container-puppet.yaml
+++ b/deployment/neutron/neutron-api-container-puppet.yaml
@@ -361,7 +361,8 @@ outputs:
             - "\n"
             - - include tripleo::profile::base::neutron::server
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerNeutronConfigImage}
+        config_image: &neutron_config_image
+            {get_param: ContainerNeutronConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/neutron_api.json:
           command:
@@ -389,6 +390,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
       docker_config:
         step_2:
           get_attr: [NeutronLogging, docker_config, step_2]
@@ -451,6 +456,12 @@ outputs:
                     environment:
                       - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
                 - {}
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'neutron'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *neutron_config_image
       host_prep_tasks: {get_attr: [NeutronLogging, host_prep_tasks]}
       metadata_settings:
         get_attr: [TLSProxyBase, role_data, metadata_settings]
diff --git a/deployment/nova/nova-api-container-puppet.yaml b/deployment/nova/nova-api-container-puppet.yaml
index c09ea378f6..834e23be5c 100644
--- a/deployment/nova/nova-api-container-puppet.yaml
+++ b/deployment/nova/nova-api-container-puppet.yaml
@@ -244,7 +244,8 @@ outputs:
             - - "['Nova_cell_v2'].each |String $val| { noop_resource($val) }"
               - include tripleo::profile::base::nova::api
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerNovaConfigImage}
+        config_image: &nova_config_image
+            {get_param: ContainerNovaConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/nova_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -253,6 +254,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -427,6 +432,12 @@ outputs:
                   - /var/lib/config-data/puppet-generated/nova/:/var/lib/kolla/config_files/src:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'nova'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *nova_config_image
       metadata_settings:
         get_attr: [ApacheServiceBase, role_data, metadata_settings]
       host_prep_tasks: {get_attr: [NovaApiLogging, host_prep_tasks]}
diff --git a/deployment/nova/nova-metadata-container-puppet.yaml b/deployment/nova/nova-metadata-container-puppet.yaml
index bbd423fc87..19bb2b1af2 100644
--- a/deployment/nova/nova-metadata-container-puppet.yaml
+++ b/deployment/nova/nova-metadata-container-puppet.yaml
@@ -186,7 +186,8 @@ outputs:
             - "\n"
             - - include tripleo::profile::base::nova::metadata
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerNovaMetadataConfigImage}
+        config_image: &nova_metadata_config_image
+            {get_param: ContainerNovaMetadataConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/nova_metadata.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -195,6 +196,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -234,6 +239,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_4:
+          config_volume: 'nova_metadata'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *nova_metadata_config_image
       host_prep_tasks: {get_attr: [NovaMetadataLogging, host_prep_tasks]}
       post_upgrade_tasks:
         - when: step|int == 1
diff --git a/deployment/octavia/octavia-api-container-puppet.yaml b/deployment/octavia/octavia-api-container-puppet.yaml
index c3e70cb16c..be01bc3b98 100644
--- a/deployment/octavia/octavia-api-container-puppet.yaml
+++ b/deployment/octavia/octavia-api-container-puppet.yaml
@@ -197,7 +197,8 @@ outputs:
             - "\n"
             - - "include tripleo::profile::base::octavia::api"
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerOctaviaConfigImage}
+        config_image: &octavia_config_image
+            {get_param: ContainerOctaviaConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/octavia_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -206,6 +207,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -289,6 +294,12 @@ outputs:
                           - ''
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'octavia'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *octavia_config_image
       host_prep_tasks:
         - name: create persistent directories
           file:
diff --git a/deployment/placement/placement-api-container-puppet.yaml b/deployment/placement/placement-api-container-puppet.yaml
index f1a9e76400..5c0bddd493 100644
--- a/deployment/placement/placement-api-container-puppet.yaml
+++ b/deployment/placement/placement-api-container-puppet.yaml
@@ -179,11 +179,20 @@ outputs:
             - "\n"
             - - {get_attr: [MySQLClient, role_data, step_config]}
               - "include tripleo::profile::base::placement::api"
-        config_image: {get_param: ContainerPlacementConfigImage}
+        config_image: &placement_config_image
+            {get_param: ContainerPlacementConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/placement_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
           config_files:
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.d"
+              dest: "/etc/httpd/conf.d"
+              merge: false
+              preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -282,6 +291,12 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'placement'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *placement_config_image
       host_prep_tasks: {get_attr: [PlacementLogging, host_prep_tasks]}
       upgrade_tasks: []
       post_upgrade_tasks:
diff --git a/deployment/swift/swift-proxy-container-puppet.yaml b/deployment/swift/swift-proxy-container-puppet.yaml
index dc0b0b5194..c083a8f46b 100644
--- a/deployment/swift/swift-proxy-container-puppet.yaml
+++ b/deployment/swift/swift-proxy-container-puppet.yaml
@@ -275,7 +275,8 @@ outputs:
         puppet_tags: swift_config,swift_proxy_config,swift_keymaster_config
         step_config: |
           include ::tripleo::profile::base::swift::proxy
-        config_image: {get_param: ContainerSwiftConfigImage}
+        config_image: &swift_config_image
+            {get_param: ContainerSwiftConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/swift_proxy.json:
           command: /usr/bin/swift-proxy-server /etc/swift/proxy-server.conf
@@ -291,6 +292,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -431,6 +436,12 @@ outputs:
                     environment:
                       - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
                 - {}
+      container_puppet_tasks:
+        step_4:
+          config_volume: 'swift'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *swift_config_image
       host_prep_tasks:
         - name: create persistent directories
           file:
diff --git a/deployment/zaqar/zaqar-container-puppet.yaml b/deployment/zaqar/zaqar-container-puppet.yaml
index 64f09f3221..a6780f815e 100644
--- a/deployment/zaqar/zaqar-container-puppet.yaml
+++ b/deployment/zaqar/zaqar-container-puppet.yaml
@@ -240,7 +240,8 @@ outputs:
             - "\n"
             - - include ::tripleo::profile::base::zaqar
               - {get_attr: [MySQLClient, role_data, step_config]}
-        config_image: {get_param: ContainerZaqarConfigImage}
+        config_image: &zaqar_config_image
+            {get_param: ContainerZaqarConfigImage}
       kolla_config:
         /var/lib/kolla/config_files/zaqar.json:
           command: /usr/sbin/httpd -DFOREGROUND
@@ -249,6 +250,10 @@ outputs:
               dest: "/etc/httpd/conf.d"
               merge: false
               preserve_properties: true
+            - source: "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d"
+              dest: "/etc/httpd/conf.modules.d"
+              merge: false
+              preserve_properties: true
             - source: "/var/lib/kolla/config_files/src/*"
               dest: "/"
               merge: true
@@ -347,6 +352,12 @@ outputs:
                       - /var/log/containers/httpd/zaqar:/var/log/httpd
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      container_puppet_tasks:
+        step_3:
+          config_volume: 'zaqar'
+          step_config: |
+            include ::tripleo::profile::base::apache
+          config_image: *zaqar_config_image
       host_prep_tasks:
         - name: create persistent directories
           file: