openstack/tripleo-common
Code Issues Proposed changes

Switch 55-heat-config to async

Browse Source 
If while a task is executing the ssh connection is severed, ansible will
automagically rerun the command under the covers. This causes problems
for long running 55-heat-config tasks as first process may have written
out the deployed json but not the notify.json that we use use to
determine if it was successful or not. This can lead to a failure
because the process either never runs to completion.  This change
switches the execution to always be run async to ensure that ssh
interruptions won't cause inconsistent failures.

We previously saw a similar issue when invoking the NetworkDeployments
using this process. We've moved the network configurations to the
NetworkConfig task in THT/common/deploy-steps.j2 but this code is still
used to invoked with OS::Heat::SoftwareDeploymentGroup

Change-Id: Ic911bb6d999caf2dc4afd4cff3d44047c03dc8e4
Related-Bug: #1792343
Closes-Bug: #1887846
This commit is contained in:
Alex Schultz 2020-07-28 14:02:53 -06:00
parent 62ed9fc219
commit 43aaaaa119
1 changed files with 2 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
tripleo_common/templates/deployments.yaml
View File

@ -178,46 +178,18 @@
become: true
when: (force | bool)
- name: Set fact for async_deployment
set_fact:
use_async_deployment: "{{ (async_deployment | default(False)) or (item == 'NetworkDeployment') }}"
- name: "Run deployment {{ item }}"
shell: |
/usr/libexec/os-refresh-config/configure.d/55-heat-config
timeout 20 bash -c -- 'while [ ! -f /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json ]; do sleep 1; done'
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json)
become: true
environment:
HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item ~ '-' ~ deployment_uuid }}
register: deployment_sync_result
ignore_errors: yes
when: not use_async_deployment|bool
- name: "Run async deployment {{ item }}"
shell: |
/usr/libexec/os-refresh-config/configure.d/55-heat-config
timeout 20 bash -c -- 'while [ ! -f /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json ]; do sleep 1; done'
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json)
become: true
environment:
HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item ~ '-' ~ deployment_uuid }}
register: deployment_async_result
ignore_errors: yes
when: use_async_deployment|bool
async: "{{ async_timeout | default(300) }}"
async: "{{ async_timeout | default(1800) }}"
poll: "{{ async_poll | default(3) }}"
- name: "Output for sync deployment {{ item }}"
debug:
msg:
- stderr: "{{ deployment_sync_result.stderr.split('\n') }}"
- status_code: "{{ deployment_sync_result.rc }}"
tags:
- output
failed_when: deployment_sync_result.rc != 0
when: not ansible_check_mode|bool and not use_async_deployment|bool
- name: "Output for async deployment {{ item }}"
debug:
msg:
@ -226,7 +198,7 @@
tags:
- output
failed_when: deployment_async_result.rc != 0
when: not ansible_check_mode|bool and use_async_deployment|bool
when: not ansible_check_mode|bool
- name: "Check-mode for Run deployment {{ item }} (changed status indicates deployment would run)"
stat: