openstack/kayobe
Code Issues Proposed changes

RabbitMQ upgrade fails due to stale /etc/hosts

Browse Source 
This bug was previously fixed but the fix did not cover all cases. This
issue is still present if an IP address other than 127.0.0.1 or the
admin IP address (but not the internal API IP address, which is correct)
is in /etc/hosts. For example, in CI we often see 127.0.1.1.

This change removes all entries from /etc/hosts for the host's current
hostname, other than the internal API IP address. It also adds a call to
rabbitmqctl status to verify that the change has worked.

Related: We ought to improve the kolla ansible prechecks to catch the
case when there is more than one IP address mapping for the host's
current hostname.

Change-Id: I2cb9928e04005c6961f3de7c571c9a06361c4f23
Story: 2003496
Task: 24773
This commit is contained in:
Mark Goddard 2018-08-20 17:04:23 +01:00
parent 27d833f13f
commit e748c1d591
1 changed files with 39 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
ansible/overcloud-etc-hosts-fixup.yml
View File

@ -7,23 +7,23 @@
# which it is listening. As a workaround, we remove the stale entries from
# /etc/hosts. See https://github.com/stackhpc/kayobe/issues/14.
- name: Ensure overcloud hosts' /etc/hosts does not contain admin network IP
- name: Ensure overcloud hosts' /etc/hosts does not contain incorrect IPs
hosts: overcloud
tags:
- etc-hosts-fixup
tasks:
- name: Ensure overcloud hosts' /etc/hosts does not contain admin network or loopback IPs
# Remove any entries from /etc/hosts that map the current hostname to an IP
# other than the host's IP on the internal API network.
- name: Ensure overcloud hosts' /etc/hosts does not contain incorrect IPs
lineinfile:
dest: /etc/hosts
regexp: "^{{ item }}[ \t]*{{ inventory_hostname }}"
regexp: "^(?!{{ internal_net_name | net_ip | regex_escape }})[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+[ \t]*{{ ansible_hostname }}"
state: absent
with_items:
- "127.0.0.1"
- "{{ admin_oc_net_name | net_ip }}"
when: admin_oc_net_name | net_ip != None
# Ensure that the correct entry is present.
validate: "grep -E '^({{ internal_net_name | net_ip | regex_escape }}).*{{ ansible_hostname }}' %s"
become: True
- name: Ensure rabbitmq containers' /etc/hosts does not contain admin network or loopback IPs
- name: Ensure rabbitmq containers' /etc/hosts does not contain incorrect IPs
hosts: overcloud
tags:
- etc-hosts-fixup
@ -32,7 +32,6 @@
- rabbitmq
- outward_rabbitmq
tasks:
- block:
- name: Check whether rabbitmq container is running
command: docker inspect -f {{ '{{.Id}}' }} {{ item }}
changed_when: False
@ -40,22 +39,28 @@
with_items: "{{ rabbitmq_containers }}"
register: ps_result
- name: Ensure rabbitmq containers' /etc/hosts does not contain admin network or loopback IPs
- name: Copy /etc/hosts into rabbitmq containers
command: docker cp /etc/hosts {{ item.item }}:/tmp/hosts
with_items: "{{ ps_result.results }}"
when: item.rc == 0
changed_when: false
- name: Ensure rabbitmq containers' /etc/hosts does not contain incorrect IPs
command: >
docker exec -u root {{ item.0.item }}
docker exec -u root {{ item.item }}
bash -c
'cp /etc/hosts /tmp/hosts &&
sed -i -e "/^{{ item.1 }}[ \t]*{{ inventory_hostname }}/d" /tmp/hosts &&
if ! diff -q /tmp/hosts /etc/hosts >/dev/null; then
'if ! diff -q /tmp/hosts /etc/hosts >/dev/null; then
cp /tmp/hosts /etc/hosts &&
echo changed
fi &&
rm /tmp/hosts'
changed_when: "'changed' in sed_result.stdout"
with_nested:
- "{{ ps_result.results }}"
- - "127.0.0.1"
- "{{ admin_oc_net_name | net_ip }}"
when: item.0.rc == 0
with_items: "{{ ps_result.results }}"
when: item.rc == 0
register: sed_result
when: admin_oc_net_name | net_ip != None
- name: Check that RabbitMQ client works
command: docker exec {{ item.item }} rabbitmqctl status
with_items: "{{ ps_result.results }}"
when: item.rc == 0
changed_when: false