[ad-hoc] Fix test case exit conditions

While working on another change, I discovered conditions
in many test cases that echoed fail messages but did not
actually exit, so the gate could succeed even though some
tests failed. This patchset aims to fix those problems, and
then fix the problems masked by those problems:

1) fix bug in revert function of file permissions module
preventing permissions from being reverted.
2) fix various syntax and logic problems in test script
3) add wait_for_tiller_ready function to avoid race condition
with test script using helm too early
4) add install for ethtool in test script
5) ignore ethtool pod failures (see note #1 in [0])
6) make logging of test results more uniform
7) Fix error message logic in perm.sh
8) Fix case in _shcommon.tpl where error message was not
logged, causing test script to unnecessarily wait for
container timeout

[0]: https://review.opendev.org/676010

Change-Id: I22182d35250c37c96e73d9f5f49abfb2246f2a35

divingbell/templates/bin/_apt.sh.tpl

@@ -59,7 +59,7 @@ load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
 
 {{- if hasKey .Values.conf "apt" }}
 {{- if hasKey .Values.conf.apt "repositories" }}
-echo -n "" > /etc/apt/trusted.gpg.d/divindbell_temp.gpg
+echo -n "" > /etc/apt/trusted.gpg.d/divingbell_temp.gpg
 echo "#The list of repositories managed by Divingbell" > /etc/apt/sources.list.divingbell
   {{- range .Values.conf.apt.repositories }}
     {{- $url := .url }}
@@ -75,16 +75,16 @@ echo "{{ printf "deb %s %s-%s %s" $url $distribution . $components }}" >>/etc/ap
       {{- end }}
     {{- end }}
     {{- if hasKey . "gpgkey" }}
-apt-key --keyring /etc/apt/trusted.gpg.d/divindbell_temp.gpg add - <<"ENDKEY"
+apt-key --keyring /etc/apt/trusted.gpg.d/divingbell_temp.gpg add - <<"ENDKEY"
 {{ .gpgkey }}
 ENDKEY
     {{- end }}
   {{- end }}
 mv /etc/apt/sources.list.divingbell /etc/apt/sources.list
 rm -rf /etc/apt/sources.list.d/*
-mv /etc/apt/trusted.gpg.d/divindbell_temp.gpg /etc/apt/trusted.gpg.d/divindbell.gpg
+mv /etc/apt/trusted.gpg.d/divingbell_temp.gpg /etc/apt/trusted.gpg.d/divingbell.gpg
 rm -f /etc/apt/trusted.gpg
-find /etc/apt/trusted.gpg.d/ -type f ! -name 'divindbell.gpg' -exec rm {{ "{}" }} \;
+find /etc/apt/trusted.gpg.d/ -type f ! -name 'divingbell.gpg' -exec rm {{ "{}" }} \;
 DEBIAN_FRONTEND=noninteractive apt-get update
 {{- end }}
 {{- if hasKey .Values.conf.apt "packages" }}

divingbell/templates/bin/_perm.sh.tpl

@@ -97,11 +97,11 @@ revert_perm(){
     for o_perm in ${revert_list}; do
       first=1
       while IFS=' ' read -r a1 a2; do
-        if [ "$first" -eq 1 && -e "$a2"]; then
-          $(chmod "$a1" "$a2")
+        if [ "$first" -eq 1 ] && [ -e "$a2" ]; then
+          chmod "$a1" "$a2"
           first=0
         elif [ -e "$a2" ]; then
-          $(chown "$a1" "$a2")
+          chown "$a1" "$a2"
         else
           log.WARN "Unable to revert permissions on $a2"
           continue
@@ -180,7 +180,7 @@ if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
   fi
  # write timestamp at beginning of execution
  log.INFO 'All permissions successfully applied on this node.'
- echo $(date +"%s") > "${hash}/last_run_timestamp"
+ date +"%s" > "${hash}/last_run_timestamp"
 
  {{- range $perm := .Values.conf.perm.paths }}
  add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
@@ -189,7 +189,7 @@ if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
 
  revert_perm
 
- if [ -n "${curr_settings}" ]; then
+ if [ -n "${applied_perm}" ]; then
   log.INFO 'All permissions successfully applied on this node.'
  else
   log.WARN 'No permissions overrides defined for this node.'

divingbell/templates/bin/_shcommon.tpl

@@ -75,7 +75,6 @@ exec >& >(while read line; do
           done)
 
 die(){
-  set +x
   # write to stderr any passed error message
   if [[ $@ = *[!\ ]* ]]; then
     log.ERROR "$@"
@@ -85,6 +84,8 @@ die(){
     log.TRACE $(caller $i)
   done
   echo __EXIT_MARKER__
+  log.ERROR "Exiting with unspecified error."
+  set +x
   # Exit after pipe closes to ensure all output is flushed first
   while : ; do
     echo "Waiting on exit..." || exit 1

doc/source/index.rst

@@ -207,7 +207,7 @@ Used to execute scripts on nodes in ``systemd`` (PID=1) namespace, for ex.::
       002-script2.sh:
         data: |
           #!/bin/bash
-          echo ${BASH_SOURCE}
+          echo ${BASH_SOURCE[0]}
       001-script1.sh:
         blocking_policy: foreground_halt_pod_on_failure
         env:
@@ -218,7 +218,7 @@ Used to execute scripts on nodes in ``systemd`` (PID=1) namespace, for ex.::
         - arg2
         data: |
           #!/bin/bash
-          echo script name: ${BASH_SOURCE}
+          echo script name: ${BASH_SOURCE[0]}
           echo args: $@
           echo env: $env1 $env2 $env3