Fix keystone fernet bootstrap

There are cases when a multinode deployment ends up in unusable
keystone public wsgi on some nodes.

The root cause is that keystone public wsgi doesn't find fernet
keys on startup - and then persists on sending 500 errors to any
requests - due to a race condition between
fernet_setup/fernet-push.sh and keystone startup.

Depends-On: https://review.opendev.org/703742/
Change-Id: I63709c2e3f6a893db82a05640da78f492bf8440f
Closes-Bug: #1846789
This commit is contained in:
Michal Nasiadka 2019-12-12 13:19:48 +01:00
parent 91c3dfe91c
commit 0799782ce8
8 changed files with 87 additions and 31 deletions

View File

@ -1,4 +1,15 @@
---
- name: Checking for any running keystone_fernet containers
become: true
kolla_container_facts:
name:
- keystone_fernet
register: container_facts
- name: Group nodes where keystone_fernet is running
group_by:
key: keystone_fernet_{{ container_facts['keystone_fernet'].State | default('bootstrap') }}
- name: Running Keystone bootstrap container
vars:
keystone: "{{ keystone_services.keystone }}"
@ -17,4 +28,29 @@
restart_policy: no
volumes: "{{ keystone.volumes|reject('equalto', '')|list }}"
run_once: True
delegate_to: "{{ groups['keystone'][0] }}"
- name: Running Keystone fernet bootstrap container
vars:
keystone_fernet: "{{ keystone_services['keystone-fernet'] }}"
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
detach: False
environment:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
image: "{{ keystone_fernet.image }}"
labels:
BOOTSTRAP:
command: >
bash -c 'sudo -E kolla_set_configs &&
keystone-manage --config-file /etc/keystone/keystone.conf
fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }} && ls -l /etc/keystone/fernet-keys/'
name: "bootstrap_keystone_fernet"
restart_policy: no
volumes: "{{ keystone_fernet.volumes|reject('equalto', '')|list }}"
run_once: True
delegate_to: "{{ groups['keystone_fernet_bootstrap'][0] }}"
when:
- keystone_token_provider == 'fernet'
- groups['keystone_fernet_running'] is not defined

View File

@ -10,7 +10,7 @@
- name: Flush handlers
meta: flush_handlers
- include_tasks: init_fernet.yml
- include_tasks: distribute_fernet.yml
when:
- keystone_token_provider == 'fernet'

View File

@ -0,0 +1,19 @@
---
- name: Waiting for Keystone SSH port to be UP
wait_for:
host: "{{ api_interface_address }}"
port: "{{ keystone_ssh_port }}"
connect_timeout: 1
register: check_keystone_ssh_port
until: check_keystone_ssh_port is success
retries: 10
delay: 5
- name: Run key distribution
become: true
command: docker exec -t keystone_fernet /usr/bin/fernet-push.sh
run_once: True
delegate_to: >-
{% if groups['keystone_fernet_running'] is defined -%}
{{ groups['keystone_fernet_running'][0] }}
{%- else -%}{{ groups['keystone'][0] }}{%- endif %}

View File

@ -1,27 +0,0 @@
---
- name: Waiting for Keystone SSH port to be UP
wait_for:
host: "{{ api_interface_address }}"
port: "{{ keystone_ssh_port }}"
connect_timeout: 1
register: check_keystone_ssh_port
until: check_keystone_ssh_port is success
retries: 10
delay: 5
- name: Initialise fernet key authentication
become: true
command: "docker exec -t keystone_fernet kolla_keystone_bootstrap {{ keystone_username }} {{ keystone_groupname }}"
register: fernet_create
changed_when: fernet_create.stdout.find('localhost | SUCCESS => ') != -1 and (fernet_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
until: fernet_create.stdout.split()[2] == 'SUCCESS' or fernet_create.stdout.find('Key repository is already initialized') != -1
retries: 10
delay: 5
run_once: True
delegate_to: "{{ groups['keystone'][0] }}"
- name: Run key distribution
become: true
command: docker exec -t keystone_fernet /usr/bin/fernet-push.sh
run_once: True
delegate_to: "{{ groups['keystone'][0] }}"

View File

@ -4,10 +4,26 @@ set -o errexit
set -o pipefail
# Get data on the fernet tokens
TOKEN_CHECK=$(/usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n {{ (groups['keystone'] | length) + 1 }})
# NOTE(mnasiadka): Check for existence of at least two tokens (should exist after bootstrap)
TOKEN_CHECK=$(/usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n 2)
# Ensure tokens are populated
n=0
while /usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q '"populated": false'; do
if [ $n -lt 10 ]; then
n=$(( n + 1 ))
echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute"
echo "DEBUG: /etc/keystone/fernet-keys contents:"
ls -l /etc/keystone/fernet-keys/
sleep 60
else
echo "CRITICAL: Waited for 10 minutes - failing"
exit 1
fi
done
# Ensure the primary token exists and is not stale
if $(echo "$TOKEN_CHECK" | grep -q '"update_required":"false"'); then
if $(echo "$TOKEN_CHECK" | grep -q '"update_required": false'); then
exit 0;
fi

View File

@ -1,5 +1,8 @@
#!/bin/bash
set -o errexit
set -o pipefail
{% for host in groups['keystone'] %}
{% if inventory_hostname != host %}
/usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys

View File

@ -1,5 +1,8 @@
#!/bin/bash
set -o errexit
set -o pipefail
keystone-manage --config-file /etc/keystone/keystone.conf fernet_rotate --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}
/usr/bin/fernet-push.sh

View File

@ -0,0 +1,6 @@
---
fixes:
- |
Rework keystone fernet bootstrap which had tendencies to fail on multinode
setups. See `bug 1846789 <https://launchpad.net/bugs/1846789>`__ for
details.