Improve tempest-cleanup ansible role

The patch implements a new flag which will fail a job when any
resources were leaked - that can be used for verification that
tests are cleaning their resources after they are finished.

Change-Id: I212cdce9713c80491153b0bbdb313e75c1f96f1d

playbooks/devstack-tempest.yaml

@@ -30,9 +30,9 @@
         name: tempest-cleanup
       vars:
         init_saved_state: true
-      when:
-        - run_tempest_dry_cleanup is defined
-        - run_tempest_cleanup is defined
+      when: (run_tempest_dry_cleanup is defined and run_tempest_dry_cleanup | bool) or
+            (run_tempest_cleanup is defined and run_tempest_cleanup | bool) or
+            (run_tempest_fail_if_leaked_resources is defined and run_tempest_fail_if_leaked_resources | bool)
 
     - name: Run Tempest
       include_role:
@@ -43,10 +43,9 @@
         name: tempest-cleanup
       vars:
         dry_run: true
-      when:
-        - run_tempest_dry_cleanup is defined
+      when: run_tempest_dry_cleanup is defined and run_tempest_dry_cleanup | bool
 
     - name: Run tempest cleanup
       include_role:
         name: tempest-cleanup
-      when: run_tempest_cleanup is defined
+      when: run_tempest_cleanup is defined and run_tempest_cleanup | bool

roles/tempest-cleanup/README.rst

@@ -31,3 +31,31 @@ saved_state.json file.
    When true, tempest cleanup creates a report (./dry_run.json) of the
    resources that would be cleaned up if the role was ran with dry_run option
    set to false.
+
+.. zuul:rolevar:: run_tempest_fail_if_leaked_resources
+   :default: false
+
+   When true, the role will fail if any leaked resources are detected.
+   The detection is done via dry_run.json file which if contains any resources,
+   some must have been leaked. This can be also used to verify that tempest
+   cleanup was successful.
+
+
+Role usage
+----------
+
+The role can be also used for verification that tempest tests don't leak any
+resources or to test that 'tempest cleanup' command deleted all leaked
+resources as expected.
+Either way the role needs to be run first with init_saved_state variable set
+to true prior any tempest tests got executed.
+Then, after tempest tests got executed this role needs to be run again with
+role variables set according to the desired outcome:
+
+1. to verify that tempest tests don't leak any resources
+   run_tempest_dry_cleanup and run_tempest_fail_if_leaked_resources have to
+   be set to true.
+
+2. to check that 'tempest cleanup' command deleted all the leaked resources
+   run_tempest_cleanup and run_tempest_fail_if_leaked_resources have to be set
+   to true.

roles/tempest-cleanup/defaults/main.yaml

@@ -1,3 +1,4 @@
 devstack_base_dir: /opt/stack
 init_saved_state: false
 dry_run: false
+run_tempest_fail_if_leaked_resources: false

roles/tempest-cleanup/tasks/dry_run.yaml

@@ -0,0 +1,7 @@
+---
+- name: Run tempest cleanup dry-run
+  become: yes
+  become_user: tempest
+  command: tox -evenv-tempest -- tempest cleanup --dry-run --debug
+  args:
+    chdir: "{{ devstack_base_dir }}/tempest"

roles/tempest-cleanup/tasks/dry_run_checker.py

@@ -0,0 +1,71 @@
+# Copyright 2020 Red Hat, Inc
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Utility for content checking of a given dry_run.json file.
+"""
+
+import argparse
+import json
+import sys
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--is-empty', action="store_true", dest='is_empty',
+                        default=False,
+                        help="""Are values of a given dry_run.json empty?""")
+    parser.add_argument('--file', dest='file', default=None, metavar='PATH',
+                        help="A path to a dry_run.json file.")
+    return parser
+
+
+def parse_arguments():
+    parser = get_parser()
+    args = parser.parse_args()
+    if not args.file:
+        sys.stderr.write('Path to a dry_run.json must be specified.\n')
+        sys.exit(1)
+    return args
+
+
+def load_json(path):
+    """Load json content from file addressed by path."""
+    try:
+        with open(path, 'rb') as json_file:
+            json_data = json.load(json_file)
+    except Exception as ex:
+        sys.exit(ex)
+    return json_data
+
+
+def are_values_empty(dry_run_content):
+    """Return true if values of dry_run.json are empty."""
+    for value in dry_run_content.values():
+        if value:
+            return False
+    return True
+
+
+def main():
+    args = parse_arguments()
+    content = load_json(args.file)
+    if args.is_empty:
+        if not are_values_empty(content):
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

roles/tempest-cleanup/tasks/main.yaml

@@ -12,20 +12,35 @@
 
 - when: dry_run
   block:
-    - name: Run tempest cleanup dry-run
-      become: yes
-      become_user: tempest
-      command: tox -evenv-tempest -- tempest cleanup --dry-run --debug
-      args:
-        chdir: "{{ devstack_base_dir }}/tempest"
+    - import_tasks: dry_run.yaml
 
     - name: Cat dry_run.json
       command: cat "{{ devstack_base_dir }}/tempest/dry_run.json"
 
-- name: Run tempest cleanup
-  become: yes
-  become_user: tempest
-  command: tox -evenv-tempest -- tempest cleanup --debug
-  args:
-    chdir: "{{ devstack_base_dir }}/tempest"
-  when: not dry_run and not init_saved_state
+- when:
+    - not dry_run
+    - not init_saved_state
+  block:
+    - name: Run tempest cleanup
+      become: yes
+      become_user: tempest
+      command: tox -evenv-tempest -- tempest cleanup --debug
+      args:
+        chdir: "{{ devstack_base_dir }}/tempest"
+
+- when:
+    - run_tempest_fail_if_leaked_resources
+    - not init_saved_state
+  block:
+    # let's run dry run again, if haven't already, to check no leftover
+    # resources were left behind after the cleanup in the previous task
+    - import_tasks: dry_run.yaml
+      when: not dry_run
+
+    - name: Fail if any resources are leaked
+      become: yes
+      become_user: tempest
+      shell: |
+        python3 roles/tempest-cleanup/tasks/dry_run_checker.py --file {{ devstack_base_dir }}/tempest/dry_run.json --is-empty
+      args:
+        chdir: "{{ devstack_base_dir }}/tempest"