Fix keystone fernet bootstrap
There are cases when a multinode deployment ends up in unusable keystone public wsgi on some nodes. The root cause is that keystone public wsgi doesn't find fernet keys on startup - and then persists on sending 500 errors to any requests - due to a race condition between fernet_setup/fernet-push.sh and keystone startup. Depends-On: https://review.opendev.org/703742/ Change-Id: I63709c2e3f6a893db82a05640da78f492bf8440f Closes-Bug: #1846789
This commit is contained in:
parent
91c3dfe91c
commit
0799782ce8
@ -1,4 +1,15 @@
|
||||
---
|
||||
- name: Checking for any running keystone_fernet containers
|
||||
become: true
|
||||
kolla_container_facts:
|
||||
name:
|
||||
- keystone_fernet
|
||||
register: container_facts
|
||||
|
||||
- name: Group nodes where keystone_fernet is running
|
||||
group_by:
|
||||
key: keystone_fernet_{{ container_facts['keystone_fernet'].State | default('bootstrap') }}
|
||||
|
||||
- name: Running Keystone bootstrap container
|
||||
vars:
|
||||
keystone: "{{ keystone_services.keystone }}"
|
||||
@ -17,4 +28,29 @@
|
||||
restart_policy: no
|
||||
volumes: "{{ keystone.volumes|reject('equalto', '')|list }}"
|
||||
run_once: True
|
||||
delegate_to: "{{ groups['keystone'][0] }}"
|
||||
|
||||
- name: Running Keystone fernet bootstrap container
|
||||
vars:
|
||||
keystone_fernet: "{{ keystone_services['keystone-fernet'] }}"
|
||||
become: true
|
||||
kolla_docker:
|
||||
action: "start_container"
|
||||
common_options: "{{ docker_common_options }}"
|
||||
detach: False
|
||||
environment:
|
||||
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
|
||||
image: "{{ keystone_fernet.image }}"
|
||||
labels:
|
||||
BOOTSTRAP:
|
||||
command: >
|
||||
bash -c 'sudo -E kolla_set_configs &&
|
||||
keystone-manage --config-file /etc/keystone/keystone.conf
|
||||
fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }} && ls -l /etc/keystone/fernet-keys/'
|
||||
name: "bootstrap_keystone_fernet"
|
||||
restart_policy: no
|
||||
volumes: "{{ keystone_fernet.volumes|reject('equalto', '')|list }}"
|
||||
run_once: True
|
||||
delegate_to: "{{ groups['keystone_fernet_bootstrap'][0] }}"
|
||||
when:
|
||||
- keystone_token_provider == 'fernet'
|
||||
- groups['keystone_fernet_running'] is not defined
|
||||
|
@ -10,7 +10,7 @@
|
||||
- name: Flush handlers
|
||||
meta: flush_handlers
|
||||
|
||||
- include_tasks: init_fernet.yml
|
||||
- include_tasks: distribute_fernet.yml
|
||||
when:
|
||||
- keystone_token_provider == 'fernet'
|
||||
|
||||
|
19
ansible/roles/keystone/tasks/distribute_fernet.yml
Normal file
19
ansible/roles/keystone/tasks/distribute_fernet.yml
Normal file
@ -0,0 +1,19 @@
|
||||
---
|
||||
- name: Waiting for Keystone SSH port to be UP
|
||||
wait_for:
|
||||
host: "{{ api_interface_address }}"
|
||||
port: "{{ keystone_ssh_port }}"
|
||||
connect_timeout: 1
|
||||
register: check_keystone_ssh_port
|
||||
until: check_keystone_ssh_port is success
|
||||
retries: 10
|
||||
delay: 5
|
||||
|
||||
- name: Run key distribution
|
||||
become: true
|
||||
command: docker exec -t keystone_fernet /usr/bin/fernet-push.sh
|
||||
run_once: True
|
||||
delegate_to: >-
|
||||
{% if groups['keystone_fernet_running'] is defined -%}
|
||||
{{ groups['keystone_fernet_running'][0] }}
|
||||
{%- else -%}{{ groups['keystone'][0] }}{%- endif %}
|
@ -1,27 +0,0 @@
|
||||
---
|
||||
- name: Waiting for Keystone SSH port to be UP
|
||||
wait_for:
|
||||
host: "{{ api_interface_address }}"
|
||||
port: "{{ keystone_ssh_port }}"
|
||||
connect_timeout: 1
|
||||
register: check_keystone_ssh_port
|
||||
until: check_keystone_ssh_port is success
|
||||
retries: 10
|
||||
delay: 5
|
||||
|
||||
- name: Initialise fernet key authentication
|
||||
become: true
|
||||
command: "docker exec -t keystone_fernet kolla_keystone_bootstrap {{ keystone_username }} {{ keystone_groupname }}"
|
||||
register: fernet_create
|
||||
changed_when: fernet_create.stdout.find('localhost | SUCCESS => ') != -1 and (fernet_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
|
||||
until: fernet_create.stdout.split()[2] == 'SUCCESS' or fernet_create.stdout.find('Key repository is already initialized') != -1
|
||||
retries: 10
|
||||
delay: 5
|
||||
run_once: True
|
||||
delegate_to: "{{ groups['keystone'][0] }}"
|
||||
|
||||
- name: Run key distribution
|
||||
become: true
|
||||
command: docker exec -t keystone_fernet /usr/bin/fernet-push.sh
|
||||
run_once: True
|
||||
delegate_to: "{{ groups['keystone'][0] }}"
|
@ -4,10 +4,26 @@ set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# Get data on the fernet tokens
|
||||
TOKEN_CHECK=$(/usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n {{ (groups['keystone'] | length) + 1 }})
|
||||
# NOTE(mnasiadka): Check for existence of at least two tokens (should exist after bootstrap)
|
||||
TOKEN_CHECK=$(/usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n 2)
|
||||
|
||||
# Ensure tokens are populated
|
||||
n=0
|
||||
while /usr/bin/python{{ distro_python_version }} /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q '"populated": false'; do
|
||||
if [ $n -lt 10 ]; then
|
||||
n=$(( n + 1 ))
|
||||
echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute"
|
||||
echo "DEBUG: /etc/keystone/fernet-keys contents:"
|
||||
ls -l /etc/keystone/fernet-keys/
|
||||
sleep 60
|
||||
else
|
||||
echo "CRITICAL: Waited for 10 minutes - failing"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Ensure the primary token exists and is not stale
|
||||
if $(echo "$TOKEN_CHECK" | grep -q '"update_required":"false"'); then
|
||||
if $(echo "$TOKEN_CHECK" | grep -q '"update_required": false'); then
|
||||
exit 0;
|
||||
fi
|
||||
|
||||
|
@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
{% for host in groups['keystone'] %}
|
||||
{% if inventory_hostname != host %}
|
||||
/usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys
|
||||
|
@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
keystone-manage --config-file /etc/keystone/keystone.conf fernet_rotate --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}
|
||||
|
||||
/usr/bin/fernet-push.sh
|
||||
|
@ -0,0 +1,6 @@
|
||||
---
|
||||
fixes:
|
||||
- |
|
||||
Rework keystone fernet bootstrap which had tendencies to fail on multinode
|
||||
setups. See `bug 1846789 <https://launchpad.net/bugs/1846789>`__ for
|
||||
details.
|
Loading…
Reference in New Issue
Block a user