diff --git a/docker/monasca/monasca-thresh/Dockerfile.j2 b/docker/monasca/monasca-thresh/Dockerfile.j2 index 554fd31f0b..acf5a99421 100644 --- a/docker/monasca/monasca-thresh/Dockerfile.j2 +++ b/docker/monasca/monasca-thresh/Dockerfile.j2 @@ -66,8 +66,14 @@ RUN cd /monasca-common-source/java \ # Overwrite the script inherited from Storm COPY extend_start.sh /usr/local/bin/kolla_extend_start + +# Add bootstrap script +COPY topology_bootstrap.sh /usr/local/bin/topology_bootstrap + RUN touch /usr/local/bin/kolla_monasca_extend_start \ - && chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_monasca_extend_start + && chmod 755 /usr/local/bin/kolla_extend_start \ + /usr/local/bin/kolla_monasca_extend_start \ + /usr/local/bin/topology_bootstrap {% block monasca_thresh_footer %}{% endblock %} {% block footer %}{% endblock %} diff --git a/docker/monasca/monasca-thresh/extend_start.sh b/docker/monasca/monasca-thresh/extend_start.sh index 6c4dda35d5..62451a8e16 100644 --- a/docker/monasca/monasca-thresh/extend_start.sh +++ b/docker/monasca/monasca-thresh/extend_start.sh @@ -42,3 +42,9 @@ if [[ $(ls -Ab ${MONASCA_WORKER_DIR}) != "" ]]; then fi . /usr/local/bin/kolla_monasca_extend_start + +# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases +# of the KOLLA_BOOTSTRAP variable being set, including empty. +if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then + . /usr/local/bin/topology_bootstrap +fi diff --git a/docker/monasca/monasca-thresh/topology_bootstrap.sh b/docker/monasca/monasca-thresh/topology_bootstrap.sh new file mode 100644 index 0000000000..923d745e59 --- /dev/null +++ b/docker/monasca/monasca-thresh/topology_bootstrap.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +# This script should be sourced by kolla_extend_start when bootstrapping +# +# Optional env(): +# TOPOLOGY_NAME("monasca-thresh") - topology name to check +# TOPOLOGY_KILL_TIMEOUT(5) - secs to wait for topology kill +# STORM_WAIT_RETRIES(24) - retries to check for storm +# STORM_WAIT_TIMEOUT(20) - secs to wait for storm list +# STORM_WAIT_DELAY(5) - secs between storm list attempts + +# - If topology exists, then: +# a) if TOPOLOGY_REPLACE is set, the existing topology is killed +# and script falls through (topology may be added) +# b) otherwise script exits with 0 (topology already exists) +# - If topology doesn't exist, script falls through (topology may be added) +# - If storm cannot be reached, or kill fails, script exits with 1 + +TOPOLOGY_NAME=${TOPOLOGY_NAME:-monasca-thresh} +TOPOLOGY_KILL_TIMEOUT=${TOPOLOGY_KILL_TIMEOUT:-5} + +# defaults from monasca-thresh +STORM_WAIT_RETRIES=${STORM_WAIT_RETRIES:-24} +STORM_WAIT_TIMEOUT=${STORM_WAIT_TIMEOUT:-20} +STORM_WAIT_DELAY=${STORM_WAIT_DELAY:-5} + +STORM="/opt/storm/bin/storm" + +echo "Waiting for storm to become available..." +success="false" +for i in $(seq "$STORM_WAIT_RETRIES"); do + if timeout "$STORM_WAIT_TIMEOUT" "$STORM" list; then + echo "Storm is available, continuing..." + success="true" + break + else + echo "Connection attempt $i of $STORM_WAIT_RETRIES failed" + sleep "$STORM_WAIT_DELAY" + fi +done + +if [ "$success" != "true" ]; then + echo "Unable to connect to Storm! Exiting..." + sleep 1 + exit 1 +fi + +locate_topology() { # + echo "Searching for topology $1 in the storm" + topologies=$("$STORM" list | awk '/-----/,0{if (!/-----/)print $1}') + found="false" + for topology in $topologies; do + if [ "$topology" = "$1" ]; then + echo "Found storm topology with name: $topology" + found="true" + break + fi + done +} + +# search for existing topology +locate_topology "$TOPOLOGY_NAME" + +if [ "$found" = "true" ]; then + + if [[ ! "${!TOPOLOGY_REPLACE[@]}" ]]; then + echo "Topology $TOPOLOGY_NAME found, submission not necessary" + exit 0 + fi + + echo "Topology replacement requested, killing old one..." + "$STORM" kill "$TOPOLOGY_NAME" -w "$TOPOLOGY_KILL_TIMEOUT" + + echo "Wait $TOPOLOGY_KILL_TIMEOUT secs for topology to reap its artifacts..." + sleep "$TOPOLOGY_KILL_TIMEOUT" + + for i in $(seq "$STORM_WAIT_RETRIES"); do + locate_topology "$TOPOLOGY_NAME" + [ "$found" != "true" ] && break + echo "... wait some more..." + sleep "$STORM_WAIT_DELAY" + done + if [ "$found" = "true" ]; then + echo "Unable to kill existing topology, giving up..." + exit 1 + fi + echo "Topology successfully killed, continuing..." +else + echo "Topology not found, continuing..." +fi diff --git a/releasenotes/notes/bug-1808805-e63af01591f03506.yaml b/releasenotes/notes/bug-1808805-e63af01591f03506.yaml new file mode 100644 index 0000000000..60597c1d01 --- /dev/null +++ b/releasenotes/notes/bug-1808805-e63af01591f03506.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Adds an option to the monasca-thresh container which checks + if the topology is currently submitted (KOLLA_BOOTSTRAP), with + an option to kill it (TOPOLOGY_REPLACE). Topology names + and various timeouts may be customized. + `LP#1808805 `__