Fix race conditions between containers

In many occasions we had log directory initialization containers
without `detach: false`, which didn't guarantee that they'll finish
before the container depending on them will start using the log
directory.

This is now fixed by moving the initialization container one global
step earlier, so that we can keep the concurrency when creating the
log dirs. (Using `detach: false` makes paunch handle just one
container at a time, and as such it can have negative performance
impact.)

For services which have their container(s) starting in step_1,
initialization cannot be moved to an earlier step, so the solution
here was to just add `detach: false`.

As a minor related change, cinder DB sync container now mounts the log
directory from host to put cinder-manage.log into the expected
location.

Change-Id: I1340de4f68dd32c2412d9385cf3a8ca202b48556
This commit is contained in:
Jiri Stransky 2017-06-14 15:24:46 +02:00
parent 0d87942dd7
commit 248099db8c
19 changed files with 43 additions and 48 deletions

View File

@ -86,16 +86,15 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
aodh_init_log:
start_order: 0
image: *aodh_image
user: root
volumes:
- /var/log/containers/aodh:/var/log/aodh
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R aodh:aodh /var/log/aodh']
step_3:
aodh_db_sync:
start_order: 1
image: *aodh_image
net: host
privileged: false

View File

@ -87,9 +87,8 @@ outputs:
owner: cinder:cinder
recurse: true
docker_config:
step_3:
step_2:
cinder_api_init_logs:
start_order: 0
image: &cinder_api_image
list_join:
- '/'
@ -99,6 +98,7 @@ outputs:
volumes:
- /var/log/containers/cinder:/var/log/cinder
command: ['/bin/bash', '-c', 'chown -R cinder:cinder /var/log/cinder']
step_3:
cinder_api_db_sync:
image: *cinder_api_image
net: host
@ -110,6 +110,7 @@ outputs:
- {get_attr: [ContainersCommon, volumes]}
-
- /var/lib/config-data/cinder/etc/cinder/:/etc/cinder/:ro
- /var/log/containers/cinder:/var/log/cinder
command:
- '/usr/bin/bootstrap_host_exec'
- 'cinder_api'

View File

@ -81,9 +81,8 @@ outputs:
owner: cinder:cinder
recurse: true
docker_config:
step_3:
step_2:
cinder_scheduler_init_logs:
start_order: 0
image: &cinder_scheduler_image
list_join:
- '/'

View File

@ -82,9 +82,8 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
congress_init_logs:
start_order: 0
image: &congress_image
list_join:
- '/'
@ -94,8 +93,8 @@ outputs:
volumes:
- /var/log/containers/congress:/var/log/congress
command: ['/bin/bash', '-c', 'chown -R congress:congress /var/log/congress']
step_3:
congress_db_sync:
start_order: 1
image: *congress_image
net: host
privileged: false

View File

@ -87,17 +87,16 @@ outputs:
recurse: true
docker_config:
# Kolla_bootstrap runs before permissions set by kolla_config
step_2:
step_1:
mysql_init_logs:
start_order: 0
image: *mysql_image
privileged: false
user: root
volumes:
- /var/log/containers/mysql:/var/log/mariadb
command: ['/bin/bash', '-c', 'chown -R mysql:mysql /var/log/mariadb']
step_2:
mysql_bootstrap:
start_order: 1
detach: false
image: *mysql_image
net: host

View File

@ -79,6 +79,7 @@ outputs:
step_1:
redis_init_logs:
start_order: 0
detach: false
image: *redis_image
privileged: false
user: root
@ -86,6 +87,7 @@ outputs:
- /var/log/containers/redis:/var/log/redis
command: ['/bin/bash', '-c', 'chown -R redis:redis /var/log/redis']
redis:
start_order: 1
image: *redis_image
net: host
privileged: false

View File

@ -85,17 +85,16 @@ outputs:
command: /usr/sbin/httpd -DFOREGROUND
docker_config:
# Kolla_bootstrap/db_sync runs before permissions set by kolla_config
step_3:
step_2:
glance_init_logs:
start_order: 0
image: *glance_image
privileged: false
user: root
volumes:
- /var/log/containers/glance:/var/log/glance
command: ['/bin/bash', '-c', 'chown -R glance:glance /var/log/glance']
step_3:
glance_api_db_sync:
start_order: 1
image: *glance_image
net: host
privileged: false

View File

@ -86,16 +86,15 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
gnocchi_init_log:
start_order: 0
image: *gnocchi_image
user: root
volumes:
- /var/log/containers/gnocchi:/var/log/gnocchi
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R gnocchi:gnocchi /var/log/gnocchi']
step_3:
gnocchi_db_sync:
start_order: 1
image: *gnocchi_image
net: host
detach: false

View File

@ -80,16 +80,15 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
heat_init_log:
start_order: 0
image: *heat_engine_image
user: root
volumes:
- /var/log/containers/heat:/var/log/heat
command: ['/bin/bash', '-c', 'chown -R heat:heat /var/log/heat']
step_3:
heat_engine_db_sync:
start_order: 1
image: *heat_engine_image
net: host
privileged: false

View File

@ -86,7 +86,7 @@ outputs:
owner: apache:apache
recurse: false
docker_config:
step_3:
step_2:
horizon_fix_perms:
image: *horizon_image
user: root
@ -99,8 +99,8 @@ outputs:
volumes:
- /var/log/containers/horizon:/var/log/horizon
- /var/lib/config-data/horizon/etc/:/etc/
step_3:
horizon:
start_order: 1
image: *horizon_image
net: host
privileged: false

View File

@ -82,9 +82,8 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
ironic_init_logs:
start_order: 0
image: &ironic_image
list_join:
- '/'
@ -94,6 +93,7 @@ outputs:
volumes:
- /var/log/containers/ironic:/var/log/ironic
command: ['/bin/bash', '-c', 'chown -R ironic:ironic /var/log/ironic']
step_3:
ironic_db_sync:
start_order: 1
image: *ironic_image

View File

@ -95,16 +95,15 @@ outputs:
command: /usr/sbin/httpd -DFOREGROUND
docker_config:
# Kolla_bootstrap/db sync runs before permissions set by kolla_config
step_3:
step_2:
keystone_init_log:
start_order: 0
image: *keystone_image
user: root
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R keystone:keystone /var/log/keystone']
volumes:
- /var/log/containers/keystone:/var/log/keystone
step_3:
keystone_db_sync:
start_order: 1
image: *keystone_image
net: host
privileged: false

View File

@ -72,6 +72,7 @@ outputs:
step_1:
memcached_init_logs:
start_order: 0
detach: false
image: *memcached_image
privileged: false
user: root

View File

@ -82,9 +82,8 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
mistral_init_logs:
start_order: 0
image: &mistral_image
list_join:
- '/'
@ -94,8 +93,9 @@ outputs:
volumes:
- /var/log/containers/mistral:/var/log/mistral
command: ['/bin/bash', '-c', 'chown -R mistral:mistral /var/log/mistral']
step_3:
mistral_db_sync:
start_order: 1
start_order: 0
image: *mistral_image
net: host
privileged: false
@ -109,7 +109,7 @@ outputs:
- /var/log/containers/mistral:/var/log/mistral
command: "/usr/bin/bootstrap_host_exec mistral_api su mistral -s /bin/bash -c 'mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head'"
mistral_db_populate:
start_order: 2
start_order: 1
image: *mistral_image
net: host
privileged: false

View File

@ -92,9 +92,8 @@ outputs:
command: /usr/sbin/httpd -DFOREGROUND
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
neutron_init_logs:
start_order: 0
image: &neutron_api_image
list_join:
- '/'
@ -104,8 +103,8 @@ outputs:
volumes:
- /var/log/containers/neutron:/var/log/neutron
command: ['/bin/bash', '-c', 'chown -R neutron:neutron /var/log/neutron']
step_3:
neutron_db_sync:
start_order: 1
image: *neutron_api_image
net: host
privileged: false

View File

@ -86,9 +86,8 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
nova_init_logs:
start_order: 0
image: &nova_api_image
list_join:
- '/'
@ -98,8 +97,9 @@ outputs:
volumes:
- /var/log/containers/nova:/var/log/nova
command: ['/bin/bash', '-c', 'chown -R nova:nova /var/log/nova']
step_3:
nova_api_db_sync:
start_order: 1
start_order: 0
image: *nova_api_image
net: host
detach: false
@ -116,7 +116,7 @@ outputs:
# to be capable of upgrading a baremetal setup. This is to ensure the name
# of the cell is 'default'
nova_api_map_cell0:
start_order: 2
start_order: 1
image: *nova_api_image
net: host
detach: false
@ -124,7 +124,7 @@ outputs:
volumes: *nova_api_volumes
command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 map_cell0'"
nova_api_create_default_cell:
start_order: 3
start_order: 2
image: *nova_api_image
net: host
detach: false
@ -136,7 +136,7 @@ outputs:
user: root
command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 create_cell --name=default'"
nova_db_sync:
start_order: 4
start_order: 3
image: *nova_api_image
net: host
detach: false

View File

@ -87,16 +87,15 @@ outputs:
owner: panko:panko
recurse: true
docker_config:
step_3:
step_2:
panko_init_log:
start_order: 0
image: *panko_image
user: root
volumes:
- /var/log/containers/panko:/var/log/panko
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R panko:panko /var/log/panko']
step_3:
panko_db_sync:
start_order: 1
image: *panko_image
net: host
detach: false

View File

@ -89,6 +89,7 @@ outputs:
step_1:
rabbitmq_init_logs:
start_order: 0
detach: false
image: *rabbitmq_image
privileged: false
user: root
@ -97,6 +98,7 @@ outputs:
command: ['/bin/bash', '-c', 'chown -R rabbitmq:rabbitmq /var/log/rabbitmq']
rabbitmq_bootstrap:
start_order: 1
detach: false
image: *rabbitmq_image
net: host
privileged: false

View File

@ -82,9 +82,8 @@ outputs:
recurse: true
docker_config:
# db sync runs before permissions set by kolla_config
step_3:
step_2:
tacker_init_logs:
start_order: 0
image: &tacker_image
list_join:
- '/'
@ -94,8 +93,8 @@ outputs:
volumes:
- /var/log/containers/tacker:/var/log/tacker
command: ['/bin/bash', '-c', 'chown -R tacker:tacker /var/log/tacker']
step_3:
tacker_db_sync:
start_order: 1
image: *tacker_image
net: host
privileged: false