Add rerun support for perm module

- Adds the ability to rerun divingbell-perm at specified interval.

- Adds the ability to specify a rerun policy of
  'always', 'never', 'once_successfully'. Default value is 'always'.

Demo: https://asciinema.org/a/220289

Change-Id: I3909b4d92f8e2bdb0d826ca1cfbd62f937c2532d

divingbell/templates/bin/_perm.sh.tpl

@@ -16,6 +16,8 @@
 # limitations under the License.
 */}}
 
+{{- $perm_loop_sleep_interval := 60 }}
+
 set -e
 
 cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/perm_host.sh
@@ -82,15 +84,10 @@ add_single_perm(){
 
 }
 
-{{- range $perm := .Values.conf.perm }}
-add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
-{{- end }}
-
-log.INFO "Applied: ${applied_perm}"
-
+revert_perm(){
 # Revert
-prev_files="$(find "${backup_path}" -type f)"
-if [ -n "${prev_files}" ]; then
+  prev_files="$(find "${backup_path}" -type f ! -name last_run_timestamp)"
+  if [ -n "${prev_files}" ]; then
     basename -a ${prev_files} | sort > /tmp/prev_perm
     echo "${applied_perm}" | sort > /tmp/curr_perm
     log.DEBUG /tmp/prev_perm
@@ -111,25 +108,103 @@ if [ -n "${prev_files}" ]; then
       rm "${backup_path}/${o_perm}"
       log.INFO "Reverted permissions and owner: ${backup_path}/${o_perm}"
     done
+  fi
+}
+
+{{- $_ := set $.Values "__rerun_policy" "always" }}
+{{- if hasKey .Values.conf "perm" }}
+{{- if hasKey .Values.conf.perm "rerun_policy" }}
+  {{- if and (not (eq .Values.conf.perm.rerun_policy "always")) (not (eq .Values.conf.perm.rerun_policy "never")) (not (eq .Values.conf.perm.rerun_policy "once_successfully")) }}
+    {{- fail (print "BAD 'rerun_policy' Got '" .Values.conf.perm.rerun_policy "', but expected 'always', 'never', or 'once_successfully'.") }}
+  {{- end }}
+  {{- $_ := set $.Values "__rerun_policy" .Values.conf.perm.rerun_policy }}
+{{- end }}
+
+{{- $_ := set $.Values "__rerun_interval" "infinite" }}
+{{- if hasKey .Values.conf.perm "rerun_interval" }}
+{{- $_ := set $.Values "__rerun_interval" .Values.conf.perm.rerun_interval }}
+
+  {{- if not (eq (.Values.conf.perm.rerun_interval | toString) "infinity") }}
+    {{- if lt (.Values.conf.perm.rerun_interval | int) $perm_loop_sleep_interval }}
+      {{- fail (print "BAD 'rerun_interval' Got '" $.Values.__rerun_interval "', but expected >= '" $perm_loop_sleep_interval "'.") }}
+    {{- end }}
+    {{- if not (eq $.Values.__rerun_policy "always") }}
+      {{- fail (print "BAD COMBINATION: Must use 'rerun_policy' of 'always' when defining a finite 'retry_interval'. Got 'rerun_policy' of '" $.Values.__rerun_policy "' and 'retry_interval' of '" $.Values.__rerun_interval "'.") }}
+    {{- end }}
+  {{- end }}
+  {{- $_ := set $.Values "__rerun_interval" .Values.conf.perm.rerun_interval }}
+{{- end }}
+
+{{- if hasKey .Values.conf.perm "rerun_policy" }}
+      {{- if and (not (eq $.Values.__rerun_policy "always")) (not (eq $.Values.__rerun_policy "never")) (not (eq $.Values.__rerun_policy "once_successfully")) }}
+        {{- fail (print "BAD 'rerun_policy' : Got '" $.Values.__rerun_policy "', but expected 'always', 'never', or 'once_successfully'.") }}
+      {{- end }}
+{{- end }}
+
+cd "${backup_path}"
+
+{{- $_ := set $.Values "__values_hash" list }}
+{{- $hash := $.Values.__values_hash | toString | sha256sum }}
+
+hash={{ $hash | squote }}
+if [ ! -d "${hash}" ]; then
+  mkdir -p "${hash}"
 fi
 
-if [ -n "${curr_settings}" ]; then
+# check rerun policy
+hash_check=fail
+if  [[ {{ $.Values.__rerun_policy }} = always ]] || \
+    [[ ! -f ${hash}/exit_code ]] || \
+   ([[ {{ $.Values.__rerun_policy }} = once_successfully ]] && \
+    [[ $(cat ${hash}/exit_code) != 0 ]]); then
+  hash_check=pass
+fi
+# check rerun interval
+interval_check=fail
+if  [[ ! -f ${hash}/last_run_timestamp ]] || [[ ! -f ${hash}/exit_code ]]; then
+  interval_check=pass
+elif [[ $(cat ${hash}/exit_code) = 0 ]]; then
+  if [[ {{ $.Values.__rerun_interval }} = infinite ]]; then
+    interval_check=pass
+  elif [[ $(date +"%s") -ge $(($(cat ${hash}/last_run_timestamp) + {{ $.Values.__rerun_interval }})) ]]; then
+    interval_check=pass
+  fi
+fi
+if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
+  if [[ -f ${hash}/exit_code ]]; then
+    # remove previous run record, in case this run is interrupted
+    rm ${hash}/exit_code
+  fi
+ # write timestamp at beginning of execution
  log.INFO 'All permissions successfully applied on this node.'
-else
+ echo $(date +"%s") > "${hash}/last_run_timestamp"
+
+ {{- range $perm := .Values.conf.perm.paths }}
+ add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
+ {{- end }}
+ log.INFO "Applied: ${applied_perm}"
+
+ revert_perm
+
+ if [ -n "${curr_settings}" ]; then
+  log.INFO 'All permissions successfully applied on this node.'
+ else
   log.WARN 'No permissions overrides defined for this node.'
+ fi
 fi
 
+echo 0 > "${hash}/exit_code"
 exit 0
+{{- end}}
 EOF
 
 chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/perm_host.sh
-chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/perm_host.sh
 
-sleep 1
-echo 'INFO Putting the daemon to sleep.'
-
-while [ 1 ]; do
-  sleep 300
+while true; do
+  chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/perm_host.sh
+  sleep 2
+  echo 'INFO Putting the daemon to sleep.'
+  sleep {{ $perm_loop_sleep_interval }}
 done
 
 exit 0

divingbell/values.yaml

@@ -32,6 +32,10 @@ conf:
     - nis
     - ntpdate
 #  perm:
+#    rerun_policy: always
+#    86400 = 1 day
+#    rerun_interval: 86400
+#    paths:
 #    -
 #      path: '/boot/System.map-*'
 #      owner: 'root'

tools/gate/scripts/020-test-divingbell.sh

@@ -435,6 +435,7 @@ test_perm(){
   local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
   echo "conf:
   perm:
+    paths:
     -
       path: ${p_test_file1}
       owner: 'root'
@@ -452,6 +453,7 @@ test_perm(){
   echo "[SUCCESS] Positive test for perm passed successfully" >> "${TEST_RESULTS}"
   echo "conf:
   perm:
+    paths:
     -
       path: ${p_test_file1}
       owner: 'root'
@@ -462,6 +464,49 @@ test_perm(){
   _test_perm_value ${p_test_file1} root shadow 640
   _test_perm_value ${p_test_file2} ${p_test_file2##*.} ${p_test_file2##*.} 777
   echo "[SUCCESS] Backup test for perm passed successfully" >> "${TEST_RESULTS}"
+  # Test invalid rerun_interval (too short)
+  echo "conf:
+  perm:
+    rerun_interval: 30
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. Got' || \
+    (echo "[FAIL] perm test invalid rerun_interval value did not receive expected 'BAD .rerun_interval. Got' error" && exit 1)
+  echo '[SUCCESS] perm test invalid rerun_interval passed successfully' >> "${TEST_RESULTS}"
+  # Test invalid rerun_interval combination
+  echo "conf:
+  perm:
+    rerun_interval: 60
+    rerun_policy: once_successfully
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
+    (echo "[FAIL] perm invalid rerun_interval combination did not receive expected 'BAD COMBINATION' error" && exit 1)
+  echo '[SUCCESS] perm invalid rerun_interval combination passed successfully' >> "${TEST_RESULTS}"
+  # test rerun_interval
+  echo "conf:
+  perm:
+    rerun_interval: 60
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}"
+  get_container_status perm
+  sleep 72
+  get_container_status perm
+  _test_perm_value ${p_test_file1} root shadow 640
+  echo '[SUCCESS] perm rerun_interval passed successfully' >> "${TEST_RESULTS}"
   _perm_teardown
 }