Use Docker healthchecks for keystone-fernet container

This change enables the use of Docker healthchecks for
keystone-fernet container. It checks if "key 0" has
right permissions, and if rsync is able to distribute
keys to other keystones.

Implements: blueprint container-health-check
Change-Id: I17bea723d4109e869cd05d211f6f8e4653f46e17

ansible/roles/keystone/defaults/main.yml

@@ -59,6 +59,7 @@ keystone_services:
       - "kolla_logs:/var/log/kolla/"
       - "keystone_fernet_tokens:/etc/keystone/fernet-keys"
     dimensions: "{{ keystone_fernet_dimensions }}"
+    healthcheck: "{{ keystone_fernet_healthcheck }}"
 
 ####################
 # Database
@@ -123,6 +124,19 @@ keystone_ssh_healthcheck:
   test: "{% if keystone_ssh_enable_healthchecks | bool %}{{ keystone_ssh_healthcheck_test }}{% else %}NONE{% endif %}"
   timeout: "{{ keystone_ssh_healthcheck_timeout }}"
 
+keystone_fernet_enable_healthchecks: "{{ enable_container_healthchecks }}"
+keystone_fernet_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
+keystone_fernet_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+keystone_fernet_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
+keystone_fernet_healthcheck_test: ["CMD-SHELL", "/usr/bin/fernet-healthcheck.sh"]
+keystone_fernet_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
+keystone_fernet_healthcheck:
+  interval: "{{ keystone_fernet_healthcheck_interval }}"
+  retries: "{{ keystone_fernet_healthcheck_retries }}"
+  start_period: "{{ keystone_fernet_healthcheck_start_period }}"
+  test: "{% if keystone_fernet_enable_healthchecks | bool %}{{ keystone_fernet_healthcheck_test }}{% else %}NONE{% endif %}"
+  timeout: "{{ keystone_fernet_healthcheck_timeout }}"
+
 keystone_default_volumes:
   - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"

ansible/roles/keystone/tasks/config.yml

@@ -223,6 +223,7 @@
     - { src: "fernet-rotate.sh.j2", dest: "fernet-rotate.sh" }
     - { src: "fernet-node-sync.sh.j2", dest: "fernet-node-sync.sh" }
     - { src: "fernet-push.sh.j2", dest: "fernet-push.sh" }
+    - { src: "fernet-healthcheck.sh.j2", dest: "fernet-healthcheck.sh" }
     - { src: "id_rsa", dest: "id_rsa" }
     - { src: "ssh_config.j2", dest: "ssh_config" }
   when:

ansible/roles/keystone/templates/fernet-healthcheck.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+(/usr/bin/fernet-node-sync.sh --check && /usr/bin/fernet-push.sh --check) || exit 1

ansible/roles/keystone/templates/fernet-node-sync.sh.j2

@@ -1,19 +1,29 @@
-#!/bin/bash
+!/bin/bash
 
 set -o errexit
 set -o pipefail
 
-# Ensure tokens are populated, check for 0 key which should always exist
-n=0
-while [ ! -f /etc/keystone/fernet-keys/0 ]; do
-    if [ $n -lt 10 ]; then
-        n=$(( n + 1 ))
-        echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute"
-        echo "DEBUG: /etc/keystone/fernet-keys contents:"
-        ls -l /etc/keystone/fernet-keys/
-        sleep 60
+if [ ! -z "$1" ] && [ "$1" == "--check" ]; then
+    if [ -f /etc/keystone/fernet-keys/0 ]; then
+        if [[ $(stat -c %U:%G /etc/keystone/fernet-keys/0) != "keystone:keystone" ]]; then
+            exit 1
+        fi
     else
-        echo "CRITICAL: Waited for 10 minutes - failing"
         exit 1
     fi
-done
+else
+    # Ensure tokens are populated, check for 0 key which should always exist
+    n=0
+    while [ ! -f /etc/keystone/fernet-keys/0 ]; do
+        if [ $n -lt 10 ]; then
+            n=$(( n + 1 ))
+            echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute"
+            echo "DEBUG: /etc/keystone/fernet-keys contents:"
+            ls -l /etc/keystone/fernet-keys/
+            sleep 60
+        else
+            echo "CRITICAL: Waited for 10 minutes - failing"
+            exit 1
+        fi
+    done
+fi

ansible/roles/keystone/templates/fernet-push.sh.j2

@@ -3,8 +3,24 @@
 set -o errexit
 set -o pipefail
 
+if [ ! -z "$1" ] && [ "$1" == "--check" ]; then
+{% if groups['keystone'] | length > 1 %}
+{% for host in groups['keystone'] %}
+{% if inventory_hostname != host %}
+/usr/bin/rsync --dry-run -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys
+{% endif %}
+{% endfor %}
+{% else %}
+echo "No additional keystone-server where fernet keys could be rsynced."
+{% endif %}
+else
+{% if groups['keystone'] | length > 1 %}
 {% for host in groups['keystone'] %}
 {% if inventory_hostname != host %}
 /usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys
 {% endif %}
 {% endfor %}
+{% else %}
+echo "No additional keystone-server where fernet keys could be rsynced."
+{% endif %}
+fi

ansible/roles/keystone/templates/keystone-fernet.json.j2

@@ -49,6 +49,12 @@
             "dest": "/etc/keystone/{{ keystone_policy_file }}",
             "owner": "keystone",
             "perm": "0600"
+        }{% endif %}{% if keystone_fernet_enable_healthchecks | bool %},
+        {
+            "source": "{{ container_config_directory }}/fernet-healthcheck.sh",
+            "dest": "/usr/bin/fernet-healthcheck.sh",
+            "owner": "root",
+            "perm": "0755"
         }{% endif %}
     ],
     "permissions": [

releasenotes/notes/implement-docker-healthchecks-for-keystone-fernet-a63033e2b95ecb2f.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Implements container healthchecks for keystone-fernet container.
+    See `blueprint
+    <https://blueprints.launchpad.net/kolla-ansible/+spec/container-health-check>`__