949d367dde
docker-puppet.py is very aggressive about running concurrently. It uses python multiprocessing to run multiple config generating containers at once. This seems to work well in general, but in some cases... perhaps when the registry is slow or under heavy load can cause timeouts to occur. Lately I'm seeing several 'container did not start before the specified timeout' errors that always seem to occur when config files are generated (docker-puppet.py is initially executed. A couple of things: -when config files are generated this is the first time most of the containers are pulled to each host machine during deployment -docker-puppet.py runs many of these processes at once. Some of them run faster, other not. -docker daemon's pull limit defaults to 3. This would throttle the above a bit perhaps contributing the the likelyhood of a timeout. One solution that seems to work for me is to set the PROCESS_COUNT in docker-puppet.py to 3. As this matches docker daemon's default it is probably safer at the cost of being slightly slower in some cases. Change-Id: I17feb3abd9d36fe7c95865a064502ce9902a074e Closes-bug: #1713188
88 lines
3.9 KiB
YAML
88 lines
3.9 KiB
YAML
# Note the indentation here is required as it's joined
|
|
# to create a playbook in deploy-steps.j2
|
|
|
|
#####################################################
|
|
# Per step puppet configuration of the baremetal host
|
|
#####################################################
|
|
- name: Write the config_step hieradata
|
|
copy: content="{{dict(step=step|int)|to_json}}" dest=/etc/puppet/hieradata/config_step.json force=true
|
|
- name: Run puppet host configuration for step {{step}}
|
|
command: >-
|
|
puppet apply
|
|
--modulepath=/etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules
|
|
--logdest syslog --logdest console --color=false
|
|
/var/lib/tripleo-config/puppet_step_config.pp
|
|
changed_when: false
|
|
check_mode: no
|
|
register: outputs
|
|
failed_when: false
|
|
no_log: true
|
|
- debug: var=(outputs.stderr|default('')).split('\n')|union(outputs.stdout_lines|default([]))
|
|
when: outputs is defined
|
|
failed_when: outputs|failed
|
|
######################################
|
|
# Generate config via docker-puppet.py
|
|
######################################
|
|
- name: Run docker-puppet tasks (generate config)
|
|
shell: python /var/lib/docker-puppet/docker-puppet.py
|
|
environment:
|
|
NET_HOST: 'true'
|
|
DEBUG: '{{docker_puppet_debug|default(false)}}'
|
|
PROCESS_COUNT: '{{docker_puppet_process_count|default(3)}}'
|
|
when: step == "1"
|
|
changed_when: false
|
|
check_mode: no
|
|
register: outputs
|
|
failed_when: false
|
|
no_log: true
|
|
- debug: var=(outputs.stderr|default('')).split('\n')|union(outputs.stdout_lines|default([]))
|
|
when: outputs is defined
|
|
failed_when: outputs|failed
|
|
##################################################
|
|
# Per step starting of the containers using paunch
|
|
##################################################
|
|
- name: Check if /var/lib/hashed-tripleo-config/docker-container-startup-config-step_{{step}}.json exists
|
|
stat:
|
|
path: /var/lib/tripleo-config/hashed-docker-container-startup-config-step_{{step}}.json
|
|
register: docker_config_json
|
|
# Note docker-puppet.py generates the hashed-*.json file, which is a copy of
|
|
# the *step_n.json with a hash of the generated external config added
|
|
# This acts as a salt to enable restarting the container if config changes
|
|
- name: Start containers for step {{step}}
|
|
command: >-
|
|
paunch --debug apply
|
|
--file /var/lib/tripleo-config/hashed-docker-container-startup-config-step_{{step}}.json
|
|
--config-id tripleo_step{{step}} --managed-by tripleo-{{role_name}}
|
|
when: docker_config_json.stat.exists
|
|
changed_when: false
|
|
check_mode: no
|
|
register: outputs
|
|
failed_when: false
|
|
no_log: true
|
|
- debug: var=(outputs.stderr|default('')).split('\n')|union(outputs.stdout_lines|default([]))
|
|
when: outputs is defined
|
|
failed_when: outputs|failed
|
|
########################################################
|
|
# Bootstrap tasks, only performed on bootstrap_server_id
|
|
########################################################
|
|
- name: Check if /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json exists
|
|
stat:
|
|
path: /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json
|
|
register: docker_puppet_tasks_json
|
|
- name: Run docker-puppet tasks (bootstrap tasks)
|
|
shell: python /var/lib/docker-puppet/docker-puppet.py
|
|
environment:
|
|
CONFIG: /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json
|
|
NET_HOST: "true"
|
|
NO_ARCHIVE: "true"
|
|
STEP: "{{step}}"
|
|
when: deploy_server_id == bootstrap_server_id and docker_puppet_tasks_json.stat.exists
|
|
changed_when: false
|
|
check_mode: no
|
|
register: outputs
|
|
failed_when: false
|
|
no_log: true
|
|
- debug: var=(outputs.stderr|default('')).split('\n')|union(outputs.stdout_lines|default([]))
|
|
when: outputs is defined
|
|
failed_when: outputs|failed
|