Switch 55-heat-config to async

If while a task is executing the ssh connection is severed, ansible will
automagically rerun the command under the covers. This causes problems
for long running 55-heat-config tasks as first process may have written
out the deployed json but not the notify.json that we use use to
determine if it was successful or not. This can lead to a failure
because the process either never runs to completion.  This change
switches the execution to always be run async to ensure that ssh
interruptions won't cause inconsistent failures.

We previously saw a similar issue when invoking the NetworkDeployments
using this process. We've moved the network configurations to the
NetworkConfig task in THT/common/deploy-steps.j2 but this code is still
used to invoked with OS::Heat::SoftwareDeploymentGroup

Change-Id: Ic911bb6d999caf2dc4afd4cff3d44047c03dc8e4
Related-Bug: #1792343
Closes-Bug: #1887846
This commit is contained in:
Alex Schultz 2020-07-28 14:02:53 -06:00
parent 62ed9fc219
commit 43aaaaa119
1 changed files with 2 additions and 30 deletions

View File

@ -178,46 +178,18 @@
become: true become: true
when: (force | bool) when: (force | bool)
- name: Set fact for async_deployment
set_fact:
use_async_deployment: "{{ (async_deployment | default(False)) or (item == 'NetworkDeployment') }}"
- name: "Run deployment {{ item }}"
shell: |
/usr/libexec/os-refresh-config/configure.d/55-heat-config
timeout 20 bash -c -- 'while [ ! -f /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json ]; do sleep 1; done'
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json)
become: true
environment:
HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item ~ '-' ~ deployment_uuid }}
register: deployment_sync_result
ignore_errors: yes
when: not use_async_deployment|bool
- name: "Run async deployment {{ item }}" - name: "Run async deployment {{ item }}"
shell: | shell: |
/usr/libexec/os-refresh-config/configure.d/55-heat-config /usr/libexec/os-refresh-config/configure.d/55-heat-config
timeout 20 bash -c -- 'while [ ! -f /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json ]; do sleep 1; done'
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json) exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json)
become: true become: true
environment: environment:
HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item ~ '-' ~ deployment_uuid }} HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item ~ '-' ~ deployment_uuid }}
register: deployment_async_result register: deployment_async_result
ignore_errors: yes ignore_errors: yes
when: use_async_deployment|bool async: "{{ async_timeout | default(1800) }}"
async: "{{ async_timeout | default(300) }}"
poll: "{{ async_poll | default(3) }}" poll: "{{ async_poll | default(3) }}"
- name: "Output for sync deployment {{ item }}"
debug:
msg:
- stderr: "{{ deployment_sync_result.stderr.split('\n') }}"
- status_code: "{{ deployment_sync_result.rc }}"
tags:
- output
failed_when: deployment_sync_result.rc != 0
when: not ansible_check_mode|bool and not use_async_deployment|bool
- name: "Output for async deployment {{ item }}" - name: "Output for async deployment {{ item }}"
debug: debug:
msg: msg:
@ -226,7 +198,7 @@
tags: tags:
- output - output
failed_when: deployment_async_result.rc != 0 failed_when: deployment_async_result.rc != 0
when: not ansible_check_mode|bool and use_async_deployment|bool when: not ansible_check_mode|bool
- name: "Check-mode for Run deployment {{ item }} (changed status indicates deployment would run)" - name: "Check-mode for Run deployment {{ item }} (changed status indicates deployment would run)"
stat: stat: