wait-for-pods: Wait for all pods to become Ready

This refactors the code inside helm-template which waits for all
pods to become Ready to move inside it's own seperate role instead
which can be used for other tasks.

Change-Id: Ibb234c46c49fe656cf918a7a4af115e0d26f23f0
This commit is contained in:
Mohammed Naser 2020-03-14 14:14:43 -04:00
parent ffe0738f60
commit 44608e16e7
5 changed files with 33 additions and 21 deletions

View File

@ -0,0 +1,4 @@
Kubernetes Roles
================
.. zuul:autorole:: wait-for-pods

View File

@ -18,4 +18,5 @@ Roles
puppet-roles
python-roles
translation-roles
kubernetes-roles
helm-roles

View File

@ -16,24 +16,6 @@
executable: /bin/bash
chdir: "{{ zuul_work_dir }}"
# NOTE(mnaser): When a StatefulSet is deployed, it creates the pods one
# by one, which means the `kubectl wait` can race if it
# is ran before the other pods are created. We instead
# check for all the StatefulSets here manually instead
# and then use the second check below to do a "confirmation"
- name: Wait for all StatefulSets to become ready
block:
- name: Retrieve all StatefulSets
command: kubectl get statefulset -o name
register: _statefulsets
- name: Ensure the number of ready replicas matches the replicas
shell: kubectl get {{ item }} -ogo-template='{{ '{{' }}eq .status.replicas .status.readyReplicas{{ '}}' }}'
register: _is_ready
until: _is_ready.stdout == 'true'
retries: 60
delay: 5
loop: "{{ _statefulsets.stdout_lines }}"
- name: Wait for all pods to become ready
command: kubectl wait --for=condition=Ready --timeout=120s pod --all
- name: Wait for all pods to become Ready
include_role:
name: wait-for-pods

View File

@ -0,0 +1,4 @@
Wait for all pods to become Ready
This role scans the current namespace for all pods and ensures that all of
them are in a Ready state.

View File

@ -0,0 +1,21 @@
# NOTE(mnaser): When a StatefulSet is deployed, it creates the pods one
# by one, which means the `kubectl wait` can race if it
# is ran before the other pods are created. We instead
# check for all the StatefulSets here manually instead
# and then use the second check below to do a "confirmation"
- name: Wait for all StatefulSets to become ready
block:
- name: Retrieve all StatefulSets
command: kubectl get statefulset -o name
register: _statefulsets
- name: Ensure the number of ready replicas matches the replicas
shell: kubectl get {{ item }} -ogo-template='{{ '{{' }}eq .status.replicas .status.readyReplicas{{ '}}' }}'
register: _is_ready
until: _is_ready.stdout == 'true'
retries: 60
delay: 5
loop: "{{ _statefulsets.stdout_lines }}"
- name: Wait for all pods to become ready
command: kubectl wait --for=condition=Ready --timeout=120s pod --all