Revert "Revert "Improve Swift healthchecks""

This reverts commit 6aa6a78fb7.

Depends-On: https://review.opendev.org/#/c/694122/
Change-Id: If455efc0702c2cfbd84b390bb9433a11e3fcebef

container-images/tripleo_kolla_template_overrides.j2

@@ -1008,15 +1008,6 @@ RUN mkdir -p /openstack && \
     chmod a+rx /openstack/healthcheck
 {% endblock %}
 
-# Note(mmagr): Below block is currently noop because swift-proxy-server kolla image is used
-#              for swift_object_expirer container, but it will allow smooth transition
-#              to usage of correct image for the service in future.
-{% block swift_object_expirer_footer %}
-RUN mkdir -p /openstack && \
-    ln -s /usr/share/openstack-tripleo-common/healthcheck/swift-object-expirer /openstack/healthcheck && \
-    chmod a+rx /openstack/healthcheck
-{% endblock %}
-
 {% block swift_proxy_server_footer %}
 RUN mkdir -p /openstack && \
     ln -s /usr/share/openstack-tripleo-common/healthcheck/swift-proxy /openstack/healthcheck && \

healthcheck/common.sh

@@ -109,3 +109,16 @@ get_url_from_vhost () {
     fi
     echo ${proto}://${server_name}:${bind_port}${wsgi_alias}
 }
+
+check_swift_interval () {
+    service=$1
+    if ps -e | grep --quiet swift-$service; then
+        interval=$(get_config_val $conf $service interval 300)
+        last=`grep -o "\"replication_last\": [0-9]*" $cache | cut -f 2 -d " "`
+        now=`date +%s`
+        if [ `expr $now - $last` -gt $interval ]; then
+            echo "Last replication run did not finish within interval of $interval seconds."
+            exit 1
+        fi
+    fi
+}

healthcheck/swift-account-replicator

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. ${HEALTHCHECK_SCRIPTS:-/usr/share/openstack-tripleo-common/healthcheck}/common.sh
+
+conf=/etc/swift/account-server.conf
+cache=/var/cache/swift/account.recon
+
+check_swift_interval account-replicator

healthcheck/swift-account-server

@@ -19,17 +19,4 @@ if ps -e | grep --quiet swift-account-server; then
         bind_host="[${bind_host}]"
     fi
     healthcheck_curl http://${bind_host}:${bind_port}/healthcheck
-else
-    if ps -e | grep --quiet swift-account-auditor; then
-        interval=$(get_config_val $conf account-auditor interval 1800)
-    elif ps -e | grep --quiet swift-account-reaper; then
-        interval=$(get_config_val $conf account-reaper interval 3600)
-    else
-        interval=$(get_config_val $conf account-replicator interval 30)
-    fi
-
-    if ! healthcheck_file_modification $cache $interval; then
-        echo "Cache file $cache was not updated within interval of $interval seconds."
-        exit 1
-    fi
 fi

healthcheck/swift-container-replicator

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. ${HEALTHCHECK_SCRIPTS:-/usr/share/openstack-tripleo-common/healthcheck}/common.sh
+
+conf=/etc/swift/container-server.conf
+cache=/var/cache/swift/container.recon
+
+check_swift_interval container-replicator

healthcheck/swift-container-server

@@ -19,17 +19,4 @@ if ps -e | grep --quiet swift-container-server; then
         bind_host="[${bind_host}]"
     fi
     healthcheck_curl http://${bind_host}:${bind_port}/healthcheck
-else
-    if ps -e | grep --quiet swift-account-auditor; then
-        interval=$(get_config_val $conf container-auditor interval 1800)
-    elif ps -e | grep --quiet swift-account-reaper; then
-        interval=$(get_config_val $conf container-replicator interval 30)
-    else
-        interval=$(get_config_val $conf container-updater interval 300)
-    fi
-
-    if ! healthcheck_file_modification $cache $interval; then
-        echo "Cache file $cache was not updated within interval of $interval seconds."
-        exit 1
-    fi
 fi

healthcheck/swift-object-expirer

@@ -1,12 +0,0 @@
-#!/bin/sh
-
-. ${HEALTHCHECK_SCRIPTS:-/usr/share/openstack-tripleo-common/healthcheck}/common.sh
-
-conf=/etc/swift/object-expirer.conf
-cache=/var/cache/swift/object.recon
-interval=$(get_config_val $conf object-expirer interval 300)
-
-if ! healthcheck_file_modification $cache $interval; then
-    echo "Cache file $cache was not updated within interval of $interval seconds."
-    exit 1
-fi

healthcheck/swift-object-replicator

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. ${HEALTHCHECK_SCRIPTS:-/usr/share/openstack-tripleo-common/healthcheck}/common.sh
+
+conf=/etc/swift/object-server.conf
+cache=/var/cache/swift/object.recon
+
+check_swift_interval object-replicator

healthcheck/swift-object-server

@@ -19,26 +19,4 @@ if ps -e | grep --quiet swift-object-server; then
         bind_host="[${bind_host}]"
     fi
     healthcheck_curl http://${bind_host}:${bind_port}/healthcheck
-elif ps -e | grep --quiet rsync; then
-    process='rsync'
-    args="${@:-873}"
-
-    if ! healthcheck_listen $process $args; then
-        ports=${args// /,}
-        echo "There is no $process process, listening on port(s) $ports, running in the container."
-        exit 1
-    fi
-else
-    if ps -e | grep --quiet swift-account-auditor; then
-        interval=$(get_config_val $conf object-auditor interval 300)
-    elif ps -e | grep --quiet swift-account-reaper; then
-        interval=$(get_config_val $conf object-replicator interval 300)
-    else
-        interval=$(get_config_val $conf object-updater interval 300)
-    fi
-
-    if ! healthcheck_file_modification $cache $interval; then
-        echo "Cache file $cache was not updated within interval of $interval seconds."
-        exit 1
-    fi
 fi

healthcheck/swift-proxy

@@ -18,15 +18,4 @@ if pgrep -f swift-proxy-server; then
         bind_host="[${bind_host}]"
     fi
     healthcheck_curl http://${bind_host}:${bind_port}/healthcheck
-else
-    # TODO(mmagr): Remove this once swift_object_expirer container will start
-    #              using swift-object-expirer kolla image
-    conf=/etc/swift/object-expirer.conf
-    cache=/var/cache/swift/object.recon
-    interval=$(get_config_val $conf object-expirer interval 300)
-
-    if ! healthcheck_file_modification $cache $interval; then
-        echo "Cache file $cache was not updated within interval of $interval seconds."
-        exit 1
-    fi
 fi

releasenotes/notes/swift-additional-healtchecks-ab8cd9c7562654f3.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds additional healtchecks for Swift to monitor account,
+    container and object replicators as well as the rsync process.