From 79851d7929027050d0d1e76aaa28e83c141dcc55 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 13 Aug 2019 07:55:58 +0000 Subject: [PATCH] Build images with healthcheck scripts for core services blueprint container-health-check Now docker supports healthcheck directive[0] in Dockerfile, which is helpful for checking container aliveness. Base for this patch were healthcheck scripts from tripleo[1]. [0] https://docs.docker.com/engine/reference/builder/#healthcheck [1] https://github.com/openstack/tripleo-common/tree/master/healthcheck Change-Id: Ifc32713cd4d6b45c74855c0ebf5bb65784701d44 --- docker/base/Dockerfile.j2 | 9 +++++++++ docker/base/healthcheck_curl | 12 ++++++++++++ docker/base/healthcheck_filemod | 15 +++++++++++++++ docker/base/healthcheck_listen | 8 ++++++++ docker/base/healthcheck_port | 8 ++++++++ docker/base/healthcheck_socket | 10 ++++++++++ docker/mariadb/Dockerfile.j2 | 9 +++++++++ docker/mariadb/healthcheck_mariadb | 17 +++++++++++++++++ kolla/common/config.py | 2 ++ kolla/image/build.py | 2 ++ 10 files changed, 92 insertions(+) create mode 100755 docker/base/healthcheck_curl create mode 100755 docker/base/healthcheck_filemod create mode 100755 docker/base/healthcheck_listen create mode 100755 docker/base/healthcheck_port create mode 100755 docker/base/healthcheck_socket create mode 100755 docker/mariadb/healthcheck_mariadb diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 590bec89e6..793471e772 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -484,6 +484,15 @@ ENTRYPOINT ["dumb-init", "--single-child", "--"] {% endif %} +{% if docker_healthchecks %} +{% block healthcheck_installation %} + +COPY healthcheck_curl healthcheck_filemod healthcheck_listen healthcheck_port healthcheck_socket /usr/local/bin/ +RUN chmod 755 /usr/local/bin/healthcheck_* + +{% endblock %} +{% endif %} + RUN touch /usr/local/bin/kolla_extend_start \ && chmod 755 /usr/local/bin/kolla_start /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_set_configs /usr/local/bin/kolla_copy_cacerts \ && chmod 440 /etc/sudoers \ diff --git a/docker/base/healthcheck_curl b/docker/base/healthcheck_curl new file mode 100755 index 0000000000..2ef8538bac --- /dev/null +++ b/docker/base/healthcheck_curl @@ -0,0 +1,12 @@ +#!/bin/bash +: ${HEALTHCHECK_CURL_MAX_TIME:=10} +: ${HEALTHCHECK_CURL_USER_AGENT:=curl-healthcheck} +: ${HEALTHCHECK_CURL_WRITE_OUT:='\n%{http_code} %{remote_ip}:%{remote_port} %{time_total} seconds\n'} +: ${HEALTHCHECK_CURL_OUTPUT:='/dev/null'} + +export NSS_SDB_USE_CACHE=no +curl -g -k -q -s -S --fail -o "${HEALTHCHECK_CURL_OUTPUT}" \ + --max-time "${HEALTHCHECK_CURL_MAX_TIME}" \ + --user-agent "${HEALTHCHECK_CURL_USER_AGENT}" \ + --write-out "${HEALTHCHECK_CURL_WRITE_OUT}" \ + "$@" || exit 1 diff --git a/docker/base/healthcheck_filemod b/docker/base/healthcheck_filemod new file mode 100755 index 0000000000..007643d9b1 --- /dev/null +++ b/docker/base/healthcheck_filemod @@ -0,0 +1,15 @@ +#!/bin/bash +file_path=$1 +limit_seconds=$2 + +# if the file doesn't exist, return 1 +if [ ! -f $file_path ]; then + echo "${file_path} does not exist for file modification check" + exit 1 +fi +curr_time=$(date +%s) +last_mod=$(stat -c '%Y' $file_path) +limit_epoch=$(( curr_time-limit_seconds )) +if [ ${limit_epoch} -gt ${last_mod} ]; then + exit 1 +fi diff --git a/docker/base/healthcheck_listen b/docker/base/healthcheck_listen new file mode 100755 index 0000000000..70273339ce --- /dev/null +++ b/docker/base/healthcheck_listen @@ -0,0 +1,8 @@ +#!/bin/bash +process=$1 + +shift 1 +args=$@ +ports=${args// /|} +pids=$(pgrep -d '|' -f $process) +ss -lnp | grep -qE ":($ports).*,pid=($pids)," diff --git a/docker/base/healthcheck_port b/docker/base/healthcheck_port new file mode 100755 index 0000000000..9531bd77a3 --- /dev/null +++ b/docker/base/healthcheck_port @@ -0,0 +1,8 @@ +#!/bin/bash +process=$1 + +shift 1 +args=$@ +ports=${args// /|} +pids=$(pgrep -d '|' -f $process) +ss -ntp | grep -qE ":($ports).*,pid=($pids)," diff --git a/docker/base/healthcheck_socket b/docker/base/healthcheck_socket new file mode 100755 index 0000000000..cd65bae69e --- /dev/null +++ b/docker/base/healthcheck_socket @@ -0,0 +1,10 @@ +#!/bin/bash +process=$1 +socket=$2 + +# lsof truncate command name to 15 characters and this behaviour +# cannot be disabled +if [ ${#process} -gt 15 ] ; then + process=${process:0:15} +fi +lsof -Fc -Ua $socket | grep -q "c$process" diff --git a/docker/mariadb/Dockerfile.j2 b/docker/mariadb/Dockerfile.j2 index 4913494596..d958b03b02 100644 --- a/docker/mariadb/Dockerfile.j2 +++ b/docker/mariadb/Dockerfile.j2 @@ -50,6 +50,15 @@ RUN mkdir -p /var/run/mysqld && chown mysql /var/run/mysqld && chmod 755 /var/ru COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh RUN chmod 755 /usr/local/bin/kolla_mariadb_backup.sh +{% if docker_healthchecks %} +{% block healthcheck_installation %} + +COPY healthcheck_mariadb /usr/local/bin/healthcheck_mariadb +RUN chmod 755 /usr/local/bin/healthcheck_mariadb + +{% endblock %} +{% endif %} + {% if use_dumb_init %} {% block mariadb_entrypoint %} # NOTE(mgoddard): Override the dumb-init arguments to avoid passing diff --git a/docker/mariadb/healthcheck_mariadb b/docker/mariadb/healthcheck_mariadb new file mode 100755 index 0000000000..b79b19bb47 --- /dev/null +++ b/docker/mariadb/healthcheck_mariadb @@ -0,0 +1,17 @@ +#!/bin/bash + +MYSQL_USERNAME="${MYSQL_USERNAME:=-haproxy}" +MYSQL_TIMEOUT=10 + +MYSQL_CMDLINE="mysql -nNE --connect-timeout=${MYSQL_TIMEOUT} -u ${MYSQL_USERNAME}" + +WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state_comment';") + +if [[ "${WSREP_STATUS}" == "Synced" ]] +then + echo "MariaDB Galera Cluster Node is synced." + exit 0 +else + echo "MariaDB Galera Cluster Node is NOT synced" + exit 0 +fi diff --git a/kolla/common/config.py b/kolla/common/config.py index 6896310109..6b5302f468 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -255,6 +255,8 @@ _CLI_OPTS = [ help='OpenStack release for building kolla-toolbox'), cfg.StrOpt('openstack-branch', default='master', help='Branch for source images'), + cfg.BoolOpt('docker-healthchecks', default=True, + help='Add Kolla docker healthcheck scripts in the image') ] _BASE_OPTS = [ diff --git a/kolla/image/build.py b/kolla/image/build.py index b29450edd8..2750d1a430 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -718,6 +718,7 @@ class KollaWorker(object): self.debian_arch = 'ppc64el' self.images = list() self.openstack_release = conf.openstack_release + self.docker_healthchecks = conf.docker_healthchecks rpm_setup_config = ([repo_file for repo_file in conf.rpm_setup_config if repo_file is not None]) self.rpm_setup = self.build_rpm_setup(rpm_setup_config) @@ -960,6 +961,7 @@ class KollaWorker(object): 'use_dumb_init': self.use_dumb_init, 'base_package_type': self.base_package_type, 'debian_arch': self.debian_arch, + 'docker_healthchecks': self.docker_healthchecks, 'supported_distro_release': supported_distro_release, 'install_metatype': self.install_metatype, 'image_prefix': self.image_prefix,