tripleo-upgrade/templates/validate_docker_images_versions.sh.j2

#!/bin/env bash
set -euo pipefail

# List of Active/Passive services separated with space
AP_SERVICES='{{ overcloud_validate_ap_services }}'
{% set overcloud_validate_ap_services_list = overcloud_validate_ap_services.split(" ") %}


function assert_image_in_file {
    #
    # Assert image used by running container is present in the registry
    # Arg 1 - container-registry file
    # Arg 2 - image to check
    #

    local _image_file=${1}
    local _image_name=${2}

    if [[ -z ${_image_file} || -z ${_image_name} ]] ; then
        echo "ERROR: Fuction assert_image_in_file requires 2 arguments"
        exit 2
    fi

    set +e
    grep -q "${_image_name}" "${_image_file}"

    if [[ ${?} -ne 0 ]] ; then
        echo "Image ${_image_name} not present in ${_image_file}"
        exit 2
    else
        set -e
        echo "Image ${_image_name} present in ${_image_file}"
    fi

}

source {{ undercloud_rc }}
for _ip in $( openstack server list -f value -c Networks | awk -F '=' '{ print $2 }' ) ; do
    echo "================================================================================"
    echo "Validate podman images at host ${_ip}"
    echo "================================================================================"
    CONTAINERS=$( ssh -q -o StrictHostKeyChecking=no \
                    {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                    "sudo podman ps 2>&1 | grep -v -e pcmk -e ID -e qrouter -e 'neutron.*ip netns exec' -e neutron-haproxy-ovnmeta | awk '{ print \$2 }'" )
    for _container in ${CONTAINERS} ; do
        # non pcs managed containers
        assert_image_in_file {{ working_dir}}/{{ container_registry_file }} ${_container}
    done

    CONTAINERS_PCMK=$( ssh -q -o StrictHostKeyChecking=no \
                        {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                        "sudo podman images 2>&1 | grep pcmk | {% if overcloud_validate_ap_services_list|length>0 %}grep -v -e {{ overcloud_validate_ap_services_list|join(' -e ') }} | {% endif %} awk '{ print \$3 }'" )

    # check if any of defined A/P services has image on a node
    if [[ -n "${AP_SERVICES}" ]] ; then
        for _ap_service in ${AP_SERVICES} ; do
            CONTAINERS_PCMK_AP=$( ssh -q -o StrictHostKeyChecking=no \
                                {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                                "sudo podman images 2>&1 | grep -e pcmk | grep ${_ap_service} | awk '{ print \$3 }'" )
            if [[ -n ${CONTAINERS_PCMK_AP} ]] ; then
                if grep -Fxq ${_ip} {{ working_dir }}/AP_SERVICES_${_ap_service}.txt; then
                   echo "ip is present, skipping"
                else
                    echo "${_ip}" >> {{ working_dir }}/AP_SERVICES_${_ap_service}.txt
                fi
            fi
        done
    fi

    if [[ -z ${CONTAINERS_PCMK} ]] ; then
        echo "PCMKLATEST images not found on ${_ip}"
    else
        echo "================================================================================"
        echo "Validate PCMKLATEST podman images at host ${_ip}"
        echo "================================================================================"
        for _img_id in ${CONTAINERS_PCMK} ; do
            _pcmk_image=$( ssh -q -o StrictHostKeyChecking=no \
                            {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                            "sudo podman images 2>&1 | grep ${_img_id} | grep -v pcmklatest | awk '{ print \$1\":\"\$2}'" )
            assert_image_in_file {{ working_dir }}/{{ container_registry_file }} ${_pcmk_image}

            _pcmk_container=$( ssh -q -o StrictHostKeyChecking=no \
                                {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                                "sudo podman ps --filter ancestor=${_img_id} | grep -v clustercheck | awk '{if (NR>1){print \$2}}'" )
            if [[ -z ${_pcmk_container} ]] ; then
                echo "PCMKLATEST container for image '${_pcmk_image}' is not running"
                echo "Running container is '${_pcmk_container}'"
                exit 2
            else
                echo "PCMKLATES container ${_pcmk_container} uses image ${_pcmk_image}"
            fi

        done
    fi

done

#
# Check for A/P services
#
for _ap_service in ${AP_SERVICES} ; do
    echo "================================================================================"
    echo "Check A/P service ${_ap_service}"
    echo "================================================================================"
    rm -f "{{ working_dir }}/${_ap_service}_found"
    if [[ -e "{{ working_dir }}/AP_SERVICES_${_ap_service}.txt" ]] ; then
        for _ip in $(cat "{{ working_dir }}/AP_SERVICES_${_ap_service}.txt") ; do
            _image=$( ssh -q -o StrictHostKeyChecking=no \
                        {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                        "sudo podman images --sort tag 2>&1" | awk '/'${_ap_service}'/ && !/pcmklatest/ { print $1":"$2}' | tail -1 )

            assert_image_in_file {{ working_dir }}/{{ container_registry_file }} ${_image}

            _container=$( ssh -q -o StrictHostKeyChecking=no \
                            {% if overcloud_ssh_user != '' %}{{overcloud_ssh_user}}{% else %}heat-admin{% endif %}@${_ip} \
                            "sudo podman ps --filter ancestor=${_ap_service} | awk '{if (NR>1){print \$2}}'" )
            if [[ -n ${_container} ]] ; then
                # so there is container running on a node
                echo "AP_PCMKLATES container ${_container} uses image ${_image} on host ${_ip}" > {{ working_dir }}/${_ap_service}_found
            fi
        done
        rm -f "{{ working_dir }}/AP_SERVICES_${_ap_service}.txt"
        if [[ ! -e {{ working_dir }}/${_ap_service}_found ]] ; then
            echo "Error: A/P service ${_ap_service} not found running on any OC node"
            exit 2
        else
            cat /home/stack/${_ap_service}_found
        fi
    fi
done