Persist containerized services httpd logs

Store the httpd logs under dedicated /var/log/containers/httpd/
paths.
Additionally, add release notes describing upgrade impact
for containerized services logs.

Closes-bug: #1700045

Change-Id: I8120c56f2315700862bd0f708b8baa8910275b09
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

docker/services/aodh-api.yaml

@@ -102,7 +102,8 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/aodh:/var/log/aodh
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R aodh:aodh /var/log/aodh']
+              - /var/log/containers/httpd/aodh-api:/var/log/httpd
+            command: ['/bin/bash', '-c', 'chown -R aodh:aodh /var/log/aodh']
         step_3:
           aodh_db_sync:
             image: *aodh_api_image
@@ -117,6 +118,7 @@ outputs:
                   - /var/lib/config-data/aodh/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
                   - /var/log/containers/aodh:/var/log/aodh
+                  - /var/log/containers/httpd/aodh-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec aodh_api su aodh -s /bin/bash -c /usr/bin/aodh-dbsync"
         step_4:
           aodh_api:
@@ -131,6 +133,7 @@ outputs:
                   - /var/lib/kolla/config_files/aodh_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/aodh/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/aodh:/var/log/aodh
+                  - /var/log/containers/httpd/aodh-api:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -146,8 +149,11 @@ outputs:
       host_prep_tasks:
       - name: create persistent logs directory
         file:
-          path: /var/log/containers/aodh
+          path: "{{ item }}"
           state: directory
+        with_items:
+          - /var/log/containers/aodh
+          - /var/log/containers/httpd/aodh-api
       upgrade_tasks:
         - name: Stop and disable aodh service (running under httpd)
           tags: step2

docker/services/cinder-api.yaml

@@ -111,6 +111,7 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/cinder:/var/log/cinder
+              - /var/log/containers/httpd/cinder-api:/var/log/httpd
             command: ['/bin/bash', '-c', 'chown -R cinder:cinder /var/log/cinder']
         step_3:
           cinder_api_db_sync:
@@ -125,6 +126,7 @@ outputs:
                 -
                   - /var/lib/config-data/cinder/etc/cinder/:/etc/cinder/:ro
                   - /var/log/containers/cinder:/var/log/cinder
+                  - /var/log/containers/httpd/cinder-api:/var/log/httpd
             command:
               - '/usr/bin/bootstrap_host_exec'
               - 'cinder_api'
@@ -145,6 +147,7 @@ outputs:
                   - /var/lib/kolla/config_files/cinder_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/cinder:/var/log/cinder
+                  - /var/log/containers/httpd/cinder-api:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -170,6 +173,7 @@ outputs:
                   - /var/lib/kolla/config_files/cinder_api_cron.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/cinder:/var/log/cinder
+                  - /var/log/containers/httpd/cinder-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
 
@@ -178,8 +182,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/cinder
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/cinder
+            - /var/log/containers/httpd/cinder-api
       upgrade_tasks:
         - name: Stop and disable cinder_api service
           tags: step2

docker/services/glance-api.yaml

@@ -118,6 +118,7 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/glance:/var/log/glance
+              - /var/log/containers/httpd/glance-api:/var/log/httpd
             command: ['/bin/bash', '-c', 'chown -R glance:glance /var/log/glance']
         step_3:
           glance_api_db_sync:
@@ -133,6 +134,7 @@ outputs:
                   - /var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json
                   - /var/lib/config-data/puppet-generated/glance_api/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/glance:/var/log/glance
+                  - /var/log/containers/httpd/glance-api:/var/log/httpd
                   - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
                   -
                     if:
@@ -176,8 +178,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/glance
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/glance
+            - /var/log/containers/httpd/glance-api
         - name: ensure ceph configurations exist
           file:
             path: /etc/ceph

docker/services/gnocchi-api.yaml

@@ -104,7 +104,8 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/gnocchi:/var/log/gnocchi
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R gnocchi:gnocchi /var/log/gnocchi']
+              - /var/log/containers/httpd/gnocchi-api:/var/log/httpd
+            command: ['/bin/bash', '-c', 'chown -R gnocchi:gnocchi /var/log/gnocchi']
         step_4:
           gnocchi_db_sync:
             image: *gnocchi_api_image
@@ -119,6 +120,7 @@ outputs:
                   - /var/lib/config-data/gnocchi/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi
+                  - /var/log/containers/httpd/gnocchi-api:/var/log/httpd
                   - /etc/ceph:/etc/ceph:ro
             command:
               str_replace:
@@ -138,6 +140,7 @@ outputs:
                   - /var/lib/kolla/config_files/gnocchi_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/gnocchi/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi
+                  - /var/log/containers/httpd/gnocchi-api:/var/log/httpd
                   - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
                   -
                     if:
@@ -154,8 +157,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/gnocchi
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/gnocchi
+            - /var/log/containers/httpd/gnocchi-api
         - name: ensure ceph configurations exist
           file:
             path: /etc/ceph

docker/services/heat-api-cfn.yaml

@@ -107,6 +107,7 @@ outputs:
                   - /var/lib/kolla/config_files/heat_api_cfn.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/heat_api_cfn/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/heat:/var/log/heat
+                  - /var/log/containers/httpd/heat-api-cfn:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -122,8 +123,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/heat
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/heat
+            - /var/log/containers/httpd/heat-api-cfn
       upgrade_tasks:
         - name: Check if heat_api_cfn is deployed
           command: systemctl is-enabled openstack-heat-api-cfn

docker/services/heat-api.yaml

@@ -118,6 +118,7 @@ outputs:
                   - /var/lib/kolla/config_files/heat_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/heat_api/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/heat:/var/log/heat
+                  - /var/log/containers/httpd/heat-api:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -143,13 +144,17 @@ outputs:
                   - /var/lib/kolla/config_files/heat_api_cron.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/heat_api/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/heat:/var/log/heat
+                  - /var/log/containers/httpd/heat-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/heat
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/heat
+            - /var/log/containers/httpd/heat-api
       upgrade_tasks:
         - name: Check is heat_api is deployed
           command: systemctl is-enabled openstack-heat-api

docker/services/horizon.yaml

@@ -110,6 +110,7 @@ outputs:
             command: ['/bin/bash', '-c', 'touch /var/log/horizon/horizon.log && chown -R apache:apache /var/log/horizon && chmod -R a+rx /etc/openstack-dashboard']
             volumes:
               - /var/log/containers/horizon:/var/log/horizon
+              - /var/log/containers/httpd/horizon:/var/log/httpd
               - /var/lib/config-data/horizon/etc/:/etc/
         step_3:
           horizon:
@@ -124,6 +125,7 @@ outputs:
                   - /var/lib/kolla/config_files/horizon.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/horizon/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/horizon:/var/log/horizon
+                  - /var/log/containers/httpd/horizon:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -139,8 +141,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/horizon
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/horizon
+            - /var/log/containers/httpd/horizon
       upgrade_tasks:
         - name: Stop and disable horizon service (running under httpd)
           tags: step2

docker/services/ironic-api.yaml

@@ -97,6 +97,7 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/ironic:/var/log/ironic
+              - /var/log/containers/httpd/ironic-api:/var/log/httpd
             command: ['/bin/bash', '-c', 'chown -R ironic:ironic /var/log/ironic']
         step_3:
           ironic_db_sync:
@@ -112,6 +113,7 @@ outputs:
                 -
                   - /var/lib/config-data/ironic_api/etc/ironic:/etc/ironic:ro
                   - /var/log/containers/ironic:/var/log/ironic
+                  - /var/log/containers/httpd/ironic-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec ironic_api su ironic -s /bin/bash -c 'ironic-dbsync --config-file /etc/ironic/ironic.conf'"
         step_4:
           ironic_api:
@@ -127,13 +129,17 @@ outputs:
                   - /var/lib/kolla/config_files/ironic_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/ironic_api/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/ironic:/var/log/ironic
+                  - /var/log/containers/httpd/ironic-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/ironic
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/ironic
+            - /var/log/containers/httpd/ironic-api
       upgrade_tasks:
         - name: Stop and disable ironic_api service
           tags: step2

docker/services/ironic-pxe.yaml

@@ -92,6 +92,7 @@ outputs:
                   - /var/lib/ironic:/var/lib/ironic/
                   - /dev/log:/dev/log
                   - /var/log/containers/ironic:/var/log/ironic
+                  - /var/log/containers/httpd/ironic-pxe:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
           ironic_pxe_http:
@@ -108,6 +109,7 @@ outputs:
                   - /var/lib/config-data/puppet-generated/ironic/:/var/lib/kolla/config_files/src:ro
                   - /var/lib/ironic:/var/lib/ironic/
                   - /var/log/containers/ironic:/var/log/ironic
+                  - /var/log/containers/httpd/ironic-pxe:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
@@ -118,3 +120,4 @@ outputs:
           with_items:
             - /var/lib/ironic
             - /var/log/containers/ironic
+            - /var/log/containers/httpd/ironic-pxe

docker/services/keystone.yaml

@@ -121,9 +121,10 @@ outputs:
           keystone_init_log:
             image: &keystone_image {get_param: DockerKeystoneImage}
             user: root
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R keystone:keystone /var/log/keystone']
+            command: ['/bin/bash', '-c', 'chown -R keystone:keystone /var/log/keystone']
             volumes:
               - /var/log/containers/keystone:/var/log/keystone
+              - /var/log/containers/httpd/keystone:/var/log/httpd
         step_3:
           keystone_db_sync:
             image: *keystone_image
@@ -138,6 +139,7 @@ outputs:
                   - /var/lib/kolla/config_files/keystone.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/keystone/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/keystone:/var/log/keystone
+                  - /var/log/containers/httpd/keystone:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -182,6 +184,7 @@ outputs:
                   - /var/lib/kolla/config_files/keystone_cron.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/keystone/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/keystone:/var/log/keystone
+                  - /var/log/containers/httpd/keystone:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       docker_puppet_tasks:
@@ -194,8 +197,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/keystone
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/keystone
+            - /var/log/containers/httpd/keystone
       upgrade_tasks:
         - name: Stop and disable keystone service (running under httpd)
           tags: step2

docker/services/manila-api.yaml

@@ -90,7 +90,8 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/manila:/var/log/manila
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R manila:manila /var/log/manila']
+              - /var/log/containers/httpd/manila-api:/var/log/httpd
+            command: ['/bin/bash', '-c', 'chown -R manila:manila /var/log/manila']
         step_3:
           manila_api_db_sync:
             user: root
@@ -103,6 +104,7 @@ outputs:
                 -
                   - /var/lib/config-data/manila/etc/manila/:/etc/manila/:ro
                   - /var/log/containers/manila:/var/log/manila
+                  - /var/log/containers/httpd/manila-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec manila_api su manila -s /bin/bash -c '/usr/bin/manila-manage db sync'"
         step_4:
           manila_api:
@@ -116,13 +118,17 @@ outputs:
                   - /var/lib/kolla/config_files/manila_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/manila/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/manila:/var/log/manila
+                  - /var/log/containers/httpd/manila-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: Create persistent manila logs directory
           file:
-            path: /var/log/containers/manila
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/manila
+            - /var/log/containers/httpd/manila-api
       upgrade_tasks:
         - name: Stop and disable manila_api service
           tags: step2

docker/services/neutron-api.yaml

@@ -110,6 +110,7 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/neutron:/var/log/neutron
+              - /var/log/containers/httpd/neutron-api:/var/log/httpd
             command: ['/bin/bash', '-c', 'chown -R neutron:neutron /var/log/neutron']
         step_3:
           neutron_db_sync:
@@ -126,6 +127,7 @@ outputs:
                   - /var/lib/config-data/neutron/etc/neutron:/etc/neutron:ro
                   - /var/lib/config-data/neutron/usr/share/neutron:/usr/share/neutron:ro
                   - /var/log/containers/neutron:/var/log/neutron
+                  - /var/log/containers/httpd/neutron-api:/var/log/httpd
             command: ['/usr/bin/bootstrap_host_exec', 'neutron_api', 'neutron-db-manage', 'upgrade', 'heads']
             # FIXME: we should make config file permissions right
             # and run as neutron user
@@ -144,6 +146,7 @@ outputs:
                       - /var/lib/kolla/config_files/neutron_api.json:/var/lib/kolla/config_files/config.json:ro
                       - /var/lib/config-data/puppet-generated/neutron/:/var/lib/kolla/config_files/src:ro
                       - /var/log/containers/neutron:/var/log/neutron
+                      - /var/log/containers/httpd/neutron-api:/var/log/httpd
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
             - if:
@@ -167,8 +170,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/neutron
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/neutron
+            - /var/log/containers/httpd/neutron-api
       upgrade_tasks:
         - name: Check if neutron_server is deployed
           command: systemctl is-enabled neutron-server

docker/services/nova-api.yaml

@@ -116,7 +116,8 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/nova:/var/log/nova
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R nova:nova /var/log/nova']
+              - /var/log/containers/httpd/nova-api:/var/log/httpd
+            command: ['/bin/bash', '-c', 'chown -R nova:nova /var/log/nova']
         step_3:
           nova_api_db_sync:
             start_order: 0
@@ -131,6 +132,7 @@ outputs:
                   - /var/lib/config-data/nova/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
                   - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
                   - /var/log/containers/nova:/var/log/nova
+                  - /var/log/containers/httpd/nova-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage api_db sync'"
           # FIXME: we probably want to wait on the 'cell_v2 update' in order for this
           # to be capable of upgrading a baremetal setup. This is to ensure the name
@@ -178,6 +180,7 @@ outputs:
                   - /var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/nova/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/nova:/var/log/nova
+                  - /var/log/containers/httpd/nova-api:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -203,6 +206,7 @@ outputs:
                   - /var/lib/kolla/config_files/nova_api_cron.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/nova/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/nova:/var/log/nova
+                  - /var/log/containers/httpd/nova-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
         step_5:
@@ -219,8 +223,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/nova
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/nova
+            - /var/log/containers/httpd/nova-api
       upgrade_tasks:
         - name: Stop and disable nova_api service
           tags: step2

docker/services/nova-placement.yaml

@@ -111,6 +111,7 @@ outputs:
                   - /var/lib/kolla/config_files/nova_placement.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/nova_placement/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/nova:/var/log/nova
+                  - /var/log/containers/httpd/nova-placement:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -128,8 +129,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/nova
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/nova
+            - /var/log/containers/httpd/nova-placement
       upgrade_tasks:
         - name: Stop and disable nova_placement service (running under httpd)
           tags: step2

docker/services/octavia-api.yaml

@@ -111,6 +111,7 @@ outputs:
               # configuration.
               - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia/
               - /var/log/containers/octavia:/var/log/octavia
+              - /var/log/containers/httpd/octavia-api:/var/log/httpd
             command: ['/bin/bash', '-c', 'mkdir -p /etc/octavia/conf.d/octavia-api; chown -R octavia:octavia /etc/octavia/conf.d/octavia-api; chown -R octavia:octavia /var/log/octavia']
         step_3:
           octavia_db_sync:
@@ -126,6 +127,7 @@ outputs:
                 -
                   - /var/lib/config-data/octavia/etc/octavia/:/etc/octavia/:ro
                   - /var/log/containers/octavia:/var/log/octavia
+                  - /var/log/containers/httpd/octavia-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec octavia_api su octavia -s /bin/bash -c '/usr/bin/octavia-db-manage upgrade head'"
         step_4:
           map_merge:
@@ -142,6 +144,7 @@ outputs:
                       - /var/lib/kolla/config_files/octavia_api.json:/var/lib/kolla/config_files/config.json:ro
                       - /var/lib/config-data/puppet-generated/octavia/:/var/lib/kolla/config_files/src:ro
                       - /var/log/containers/octavia:/var/log/octavia
+                      - /var/log/containers/httpd/octavia-api:/var/log/httpd
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
             - if:
@@ -166,8 +169,11 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/octavia
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/octavia
+            - /var/log/containers/httpd/octavia-api
       upgrade_tasks:
         - name: Stop and disable octavia_api service
           tags: step2

docker/services/panko-api.yaml

@@ -104,7 +104,8 @@ outputs:
             user: root
             volumes:
               - /var/log/containers/panko:/var/log/panko
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R panko:panko /var/log/panko']
+              - /var/log/containers/httpd/panko-api:/var/log/httpd
+            command: ['/bin/bash', '-c', 'chown -R panko:panko /var/log/panko']
         step_3:
           panko_db_sync:
             image: *panko_api_image
@@ -119,6 +120,7 @@ outputs:
                   - /var/lib/config-data/panko/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
                   - /var/lib/config-data/panko/etc/panko:/etc/panko:ro
                   - /var/log/containers/panko:/var/log/panko
+                  - /var/log/containers/httpd/panko-api:/var/log/httpd
             command: "/usr/bin/bootstrap_host_exec panko_api su panko -s /bin/bash -c '/usr/bin/panko-dbsync'"
         step_4:
           panko_api:
@@ -134,6 +136,7 @@ outputs:
                   - /var/lib/kolla/config_files/panko_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/panko/:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/panko:/var/log/panko
+                  - /var/log/containers/httpd/panko-api:/var/log/httpd
                   -
                     if:
                       - internal_tls_enabled
@@ -149,7 +152,10 @@ outputs:
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/panko
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/panko
+            - /var/log/containers/httpd/panko-api
       metadata_settings:
         get_attr: [PankoApiPuppetBase, role_data, metadata_settings]

docker/services/swift-proxy.yaml

@@ -111,6 +111,7 @@ outputs:
                       - /srv/node:/srv/node
                       - /dev:/dev
                       - /var/log/containers/swift:/var/log/swift
+                      - /var/log/containers/httpd/swift-proxy:/var/log/httpd
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
             - if:
@@ -138,6 +139,7 @@ outputs:
             state: directory
           with_items:
             - /var/log/containers/swift
+            - /var/log/containers/httpd/swift-proxy
             - /srv/node
       upgrade_tasks:
         - name: Stop and disable swift_proxy service

docker/services/zaqar.yaml

@@ -116,6 +116,7 @@ outputs:
                   user: root
                   volumes:
                     - /var/log/containers/zaqar:/var/log/zaqar
+                    - /var/log/containers/httpd/zaqar:/var/log/httpd
                   command: ['/bin/bash', '-c', 'chown -R zaqar:zaqar /var/log/zaqar']
               step_3:
                 zaqar_db_sync:
@@ -130,6 +131,7 @@ outputs:
                       -
                         - /var/lib/config-data/zaqar/etc/zaqar/:/etc/zaqar/:ro
                         - /var/log/containers/zaqar:/var/log/zaqar
+                        - /var/log/containers/httpd/zaqar:/var/log/httpd
                   command: "/usr/bin/bootstrap_host_exec zaqar su zaqar -s /bin/bash -c 'zaqar-sql-db-manage upgrade head'"
             - {}
           - step_4:
@@ -148,6 +150,7 @@ outputs:
                       - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
                       - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
                       - /var/log/containers/zaqar:/var/log/zaqar
+                      - /var/log/containers/httpd/zaqar:/var/log/httpd
                       -
                         if:
                           - internal_tls_enabled
@@ -172,13 +175,17 @@ outputs:
                       - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
                       - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
                       - /var/log/containers/zaqar:/var/log/zaqar
+                      - /var/log/containers/httpd/zaqar:/var/log/httpd
                 environment:
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: create persistent logs directory
           file:
-            path: /var/log/containers/zaqar
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/zaqar
+            - /var/log/containers/httpd/zaqar
       upgrade_tasks:
         - name: Stop and disable zaqar service
           tags: step2

releasenotes/notes/containerized-services-logs-0dc652513870f46d.yaml

@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    Containerized services logs can be found under updated paths.
+    Pacemaker-managed resources write logs to `/var/log/pacemaker/bundles/*`.
+    Docker-daemon managed openstack services bind-mount their log files to the
+    `/var/log/containers/<foo>/*` sub-directories. Services running under
+    Apache2 WSGI use the `/var/log/containers/httpd/<foo-api>/*` destinations.
+    Additional tools or commands that log to syslog, end up placing log records
+    into the hosts journalctl and `/var/log/messages`.
+