Fix race conditions between containers

In many occasions we had log directory initialization containers
without `detach: false`, which didn't guarantee that they'll finish
before the container depending on them will start using the log
directory.

This is now fixed by moving the initialization container one global
step earlier, so that we can keep the concurrency when creating the
log dirs. (Using `detach: false` makes paunch handle just one
container at a time, and as such it can have negative performance
impact.)

For services which have their container(s) starting in step_1,
initialization cannot be moved to an earlier step, so the solution
here was to just add `detach: false`.

As a minor related change, cinder DB sync container now mounts the log
directory from host to put cinder-manage.log into the expected
location.

Change-Id: I1340de4f68dd32c2412d9385cf3a8ca202b48556

docker/services/aodh-api.yaml

@@ -86,16 +86,15 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           aodh_init_log:
-            start_order: 0
             image: *aodh_image
             user: root
             volumes:
               - /var/log/containers/aodh:/var/log/aodh
             command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R aodh:aodh /var/log/aodh']
+        step_3:
           aodh_db_sync:
-            start_order: 1
             image: *aodh_image
             net: host
             privileged: false

docker/services/cinder-api.yaml

@@ -87,9 +87,8 @@ outputs:
               owner: cinder:cinder
               recurse: true
       docker_config:
-        step_3:
+        step_2:
           cinder_api_init_logs:
-            start_order: 0
             image: &cinder_api_image
               list_join:
                 - '/'
@@ -99,6 +98,7 @@ outputs:
             volumes:
               - /var/log/containers/cinder:/var/log/cinder
             command: ['/bin/bash', '-c', 'chown -R cinder:cinder /var/log/cinder']
+        step_3:
           cinder_api_db_sync:
             image: *cinder_api_image
             net: host
@@ -110,6 +110,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/cinder/etc/cinder/:/etc/cinder/:ro
+                  - /var/log/containers/cinder:/var/log/cinder
             command:
               - '/usr/bin/bootstrap_host_exec'
               - 'cinder_api'

docker/services/cinder-scheduler.yaml

@@ -81,9 +81,8 @@ outputs:
               owner: cinder:cinder
               recurse: true
       docker_config:
-        step_3:
+        step_2:
           cinder_scheduler_init_logs:
-            start_order: 0
             image: &cinder_scheduler_image
               list_join:
                 - '/'

docker/services/congress-api.yaml

@@ -82,9 +82,8 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           congress_init_logs:
-            start_order: 0
             image: &congress_image
               list_join:
                 - '/'
@@ -94,8 +93,8 @@ outputs:
             volumes:
               - /var/log/containers/congress:/var/log/congress
             command: ['/bin/bash', '-c', 'chown -R congress:congress /var/log/congress']
+        step_3:
           congress_db_sync:
-            start_order: 1
             image: *congress_image
             net: host
             privileged: false

docker/services/database/mysql.yaml

@@ -87,17 +87,16 @@ outputs:
               recurse: true
       docker_config:
         # Kolla_bootstrap runs before permissions set by kolla_config
-        step_2:
+        step_1:
           mysql_init_logs:
-            start_order: 0
             image: *mysql_image
             privileged: false
             user: root
             volumes:
               - /var/log/containers/mysql:/var/log/mariadb
             command: ['/bin/bash', '-c', 'chown -R mysql:mysql /var/log/mariadb']
+        step_2:
           mysql_bootstrap:
-            start_order: 1
             detach: false
             image: *mysql_image
             net: host

docker/services/database/redis.yaml

@@ -79,6 +79,7 @@ outputs:
         step_1:
           redis_init_logs:
             start_order: 0
+            detach: false
             image: *redis_image
             privileged: false
             user: root
@@ -86,6 +87,7 @@ outputs:
               - /var/log/containers/redis:/var/log/redis
             command: ['/bin/bash', '-c', 'chown -R redis:redis /var/log/redis']
           redis:
+            start_order: 1
             image: *redis_image
             net: host
             privileged: false

docker/services/glance-api.yaml

@@ -85,17 +85,16 @@ outputs:
           command: /usr/sbin/httpd -DFOREGROUND
       docker_config:
         # Kolla_bootstrap/db_sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           glance_init_logs:
-            start_order: 0
             image: *glance_image
             privileged: false
             user: root
             volumes:
               - /var/log/containers/glance:/var/log/glance
             command: ['/bin/bash', '-c', 'chown -R glance:glance /var/log/glance']
+        step_3:
           glance_api_db_sync:
-            start_order: 1
             image: *glance_image
             net: host
             privileged: false

docker/services/gnocchi-api.yaml

@@ -86,16 +86,15 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           gnocchi_init_log:
-            start_order: 0
             image: *gnocchi_image
             user: root
             volumes:
               - /var/log/containers/gnocchi:/var/log/gnocchi
             command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R gnocchi:gnocchi /var/log/gnocchi']
+        step_3:
           gnocchi_db_sync:
-            start_order: 1
             image: *gnocchi_image
             net: host
             detach: false

docker/services/heat-engine.yaml

@@ -80,16 +80,15 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           heat_init_log:
-            start_order: 0
             image: *heat_engine_image
             user: root
             volumes:
               - /var/log/containers/heat:/var/log/heat
             command: ['/bin/bash', '-c', 'chown -R heat:heat /var/log/heat']
+        step_3:
           heat_engine_db_sync:
-            start_order: 1
             image: *heat_engine_image
             net: host
             privileged: false

docker/services/horizon.yaml

@@ -86,7 +86,7 @@ outputs:
               owner: apache:apache
               recurse: false
       docker_config:
-        step_3:
+        step_2:
           horizon_fix_perms:
             image: *horizon_image
             user: root
@@ -99,8 +99,8 @@ outputs:
             volumes:
               - /var/log/containers/horizon:/var/log/horizon
               - /var/lib/config-data/horizon/etc/:/etc/
+        step_3:
           horizon:
-            start_order: 1
             image: *horizon_image
             net: host
             privileged: false

docker/services/ironic-api.yaml

@@ -82,9 +82,8 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           ironic_init_logs:
-            start_order: 0
             image: &ironic_image
               list_join:
                 - '/'
@@ -94,6 +93,7 @@ outputs:
             volumes:
               - /var/log/containers/ironic:/var/log/ironic
             command: ['/bin/bash', '-c', 'chown -R ironic:ironic /var/log/ironic']
+        step_3:
           ironic_db_sync:
             start_order: 1
             image: *ironic_image

docker/services/keystone.yaml

@@ -95,16 +95,15 @@ outputs:
           command: /usr/sbin/httpd -DFOREGROUND
       docker_config:
         # Kolla_bootstrap/db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           keystone_init_log:
-            start_order: 0
             image: *keystone_image
             user: root
             command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R keystone:keystone /var/log/keystone']
             volumes:
               - /var/log/containers/keystone:/var/log/keystone
+        step_3:
           keystone_db_sync:
-            start_order: 1
             image: *keystone_image
             net: host
             privileged: false

docker/services/memcached.yaml

@@ -72,6 +72,7 @@ outputs:
         step_1:
           memcached_init_logs:
             start_order: 0
+            detach: false
             image: *memcached_image
             privileged: false
             user: root

docker/services/mistral-api.yaml

@@ -82,9 +82,8 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           mistral_init_logs:
-            start_order: 0
             image: &mistral_image
               list_join:
                 - '/'
@@ -94,8 +93,9 @@ outputs:
             volumes:
               - /var/log/containers/mistral:/var/log/mistral
             command: ['/bin/bash', '-c', 'chown -R mistral:mistral /var/log/mistral']
+        step_3:
           mistral_db_sync:
-            start_order: 1
+            start_order: 0
             image: *mistral_image
             net: host
             privileged: false
@@ -109,7 +109,7 @@ outputs:
                   - /var/log/containers/mistral:/var/log/mistral
             command: "/usr/bin/bootstrap_host_exec mistral_api su mistral -s /bin/bash -c 'mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head'"
           mistral_db_populate:
-            start_order: 2
+            start_order: 1
             image: *mistral_image
             net: host
             privileged: false

docker/services/neutron-api.yaml

@@ -92,9 +92,8 @@ outputs:
           command: /usr/sbin/httpd -DFOREGROUND
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           neutron_init_logs:
-            start_order: 0
             image: &neutron_api_image
               list_join:
                 - '/'
@@ -104,8 +103,8 @@ outputs:
             volumes:
               - /var/log/containers/neutron:/var/log/neutron
             command: ['/bin/bash', '-c', 'chown -R neutron:neutron /var/log/neutron']
+        step_3:
           neutron_db_sync:
-            start_order: 1
             image: *neutron_api_image
             net: host
             privileged: false

docker/services/nova-api.yaml

@@ -86,9 +86,8 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           nova_init_logs:
-            start_order: 0
             image: &nova_api_image
               list_join:
                 - '/'
@@ -98,8 +97,9 @@ outputs:
             volumes:
               - /var/log/containers/nova:/var/log/nova
             command: ['/bin/bash', '-c', 'chown -R nova:nova /var/log/nova']
+        step_3:
           nova_api_db_sync:
-            start_order: 1
+            start_order: 0
             image: *nova_api_image
             net: host
             detach: false
@@ -116,7 +116,7 @@ outputs:
           # to be capable of upgrading a baremetal setup. This is to ensure the name
           # of the cell is 'default'
           nova_api_map_cell0:
-            start_order: 2
+            start_order: 1
             image: *nova_api_image
             net: host
             detach: false
@@ -124,7 +124,7 @@ outputs:
             volumes: *nova_api_volumes
             command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 map_cell0'"
           nova_api_create_default_cell:
-            start_order: 3
+            start_order: 2
             image: *nova_api_image
             net: host
             detach: false
@@ -136,7 +136,7 @@ outputs:
             user: root
             command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 create_cell --name=default'"
           nova_db_sync:
-            start_order: 4
+            start_order: 3
             image: *nova_api_image
             net: host
             detach: false

docker/services/panko-api.yaml

@@ -87,16 +87,15 @@ outputs:
               owner: panko:panko
               recurse: true
       docker_config:
-        step_3:
+        step_2:
           panko_init_log:
-            start_order: 0
             image: *panko_image
             user: root
             volumes:
               - /var/log/containers/panko:/var/log/panko
             command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R panko:panko /var/log/panko']
+        step_3:
           panko_db_sync:
-            start_order: 1
             image: *panko_image
             net: host
             detach: false

docker/services/rabbitmq.yaml

@@ -89,6 +89,7 @@ outputs:
         step_1:
           rabbitmq_init_logs:
             start_order: 0
+            detach: false
             image: *rabbitmq_image
             privileged: false
             user: root
@@ -97,6 +98,7 @@ outputs:
             command: ['/bin/bash', '-c', 'chown -R rabbitmq:rabbitmq /var/log/rabbitmq']
           rabbitmq_bootstrap:
             start_order: 1
+            detach: false
             image: *rabbitmq_image
             net: host
             privileged: false

docker/services/tacker.yaml

@@ -82,9 +82,8 @@ outputs:
               recurse: true
       docker_config:
         # db sync runs before permissions set by kolla_config
-        step_3:
+        step_2:
           tacker_init_logs:
-            start_order: 0
             image: &tacker_image
               list_join:
                 - '/'
@@ -94,8 +93,8 @@ outputs:
             volumes:
               - /var/log/containers/tacker:/var/log/tacker
             command: ['/bin/bash', '-c', 'chown -R tacker:tacker /var/log/tacker']
+        step_3:
           tacker_db_sync:
-            start_order: 1
             image: *tacker_image
             net: host
             privileged: false