Allow retry when wc_notify not successed

Task: 36946

Depends-On: https://review.opendev.org/#/c/700512/

Change-Id: I0689ac7e434b6e621e8d3a3be9d3420eea570e68

heat_tempest_plugin/tests/functional/test_os_wait_condition.py

@@ -45,31 +45,41 @@ resources:
         str_replace:
           template: '#!/bin/sh
 
-            wc_notify --data-binary ''{"status": "SUCCESS"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
 
             # signals with reason
 
-            wc_notify --data-binary ''{"status": "SUCCESS", "reason":
-            "signal2"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS", "reason":
+            "signal2"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
 
             # signals with data
 
-            wc_notify --data-binary ''{"status": "SUCCESS", "reason":
-            "signal3", "data": "data3"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS", "reason": "signal3", "data":
+            "data3"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
 
-            wc_notify --data-binary ''{"status": "SUCCESS", "reason":
-            "signal4", "data": "data4"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS", "reason": "signal4", "data":
+            "data4"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
 
             # check signals with the same ID
 
-            wc_notify --data-binary ''{"status": "SUCCESS", "id": "test5"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS", "id":
+            "test5"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
 
-            wc_notify --data-binary ''{"status": "SUCCESS", "id": "test5"}''
+            while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS", "id":
+            "test5"}''; if [ $? -eq 0 ]; then break; else sleep 10; fi; done
+
+            _signal(){ while true; do wc_notify --data-binary ''{"status":
+            "SUCCESS"}''; if [ $? -eq 0 ]; then break; fi; done }
 
             # loop for 20 signals without reasons and data
 
-            for i in `seq 1 20`; do wc_notify --data-binary ''{"status":
-            "SUCCESS"}'' & done
+            for i in `seq 1 20`; do _signal & done
 
             wait
             '

heat_tempest_plugin/tests/scenario/templates/lb_member.yaml

@@ -35,8 +35,16 @@ resources:
             #! /bin/sh -v
             Body=$(hostname)
             Response="HTTP/1.1 200 OK\r\nContent-Length: ${#Body}\r\n\r\n$Body"
-            wc_notify --data-binary '{"status": "SUCCESS"}'
-            while true ; do echo -e $Response | nc -llp PORT; done
+            while true; do
+                wc_notify --data-binary '{"status": "SUCCESS"}'
+                if [ $? -eq 0 ]; then
+                    break
+                fi
+                sleep 10
+            done
+            while true ; do
+                echo -e $Response | nc -llp PORT
+            done
           params:
             PORT: {get_param: app_port}
             wc_notify: { get_attr: [handle, curl_cli]}

heat_tempest_plugin/tests/scenario/templates/netcat-webapp.yaml

@@ -24,8 +24,16 @@ resources:
             #! /bin/sh -v
             Body=$(hostname)
             Response="HTTP/1.1 200 OK\r\nContent-Length: ${#Body}\r\n\r\n$Body"
-            wc_notify --data-binary '{"status": "SUCCESS"}'
-            while true ; do echo -e $Response | nc -llp PORT; done
+            while true; do
+                wc_notify --data-binary '{"status": "SUCCESS"}'
+                if [ $? -eq 0 ]; then
+                    break
+                fi
+                sleep 10
+            done
+            while true ; do
+                echo -e $Response | nc -llp PORT
+            done
           params:
             PORT: { get_param: app_port }
             wc_notify: { get_param: wc_curl_cli }

heat_tempest_plugin/tests/scenario/templates/test_server_signal.yaml

@@ -87,7 +87,13 @@ resources:
         str_replace:
           template: |
             #!/bin/sh
-            wc_notify --data-binary '{"status": "SUCCESS", "data": "test complete"}'
+            while true; do
+                wc_notify --data-binary '{"status": "SUCCESS", "data": "test complete"}'
+                if [ $? -eq 0 ]; then
+                    break
+                fi
+                sleep 10
+            done
           params:
             wc_notify:
               list_join: