From 05c35eb7932171e2f86a47fa72cad15c4d3dfebf Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Wed, 12 Aug 2020 09:17:35 +0000 Subject: [PATCH] Improve tempest-cleanup ansible role The patch implements a new flag which will fail a job when any resources were leaked - that can be used for verification that tests are cleaning their resources after they are finished. Change-Id: I212cdce9713c80491153b0bbdb313e75c1f96f1d --- playbooks/devstack-tempest.yaml | 11 ++- roles/tempest-cleanup/README.rst | 28 ++++++++ roles/tempest-cleanup/defaults/main.yaml | 1 + roles/tempest-cleanup/tasks/dry_run.yaml | 7 ++ .../tempest-cleanup/tasks/dry_run_checker.py | 71 +++++++++++++++++++ roles/tempest-cleanup/tasks/main.yaml | 41 +++++++---- 6 files changed, 140 insertions(+), 19 deletions(-) create mode 100644 roles/tempest-cleanup/tasks/dry_run.yaml create mode 100644 roles/tempest-cleanup/tasks/dry_run_checker.py diff --git a/playbooks/devstack-tempest.yaml b/playbooks/devstack-tempest.yaml index 7ee74110d4..4539bf90e3 100644 --- a/playbooks/devstack-tempest.yaml +++ b/playbooks/devstack-tempest.yaml @@ -30,9 +30,9 @@ name: tempest-cleanup vars: init_saved_state: true - when: - - run_tempest_dry_cleanup is defined - - run_tempest_cleanup is defined + when: (run_tempest_dry_cleanup is defined and run_tempest_dry_cleanup | bool) or + (run_tempest_cleanup is defined and run_tempest_cleanup | bool) or + (run_tempest_fail_if_leaked_resources is defined and run_tempest_fail_if_leaked_resources | bool) - name: Run Tempest include_role: @@ -43,10 +43,9 @@ name: tempest-cleanup vars: dry_run: true - when: - - run_tempest_dry_cleanup is defined + when: run_tempest_dry_cleanup is defined and run_tempest_dry_cleanup | bool - name: Run tempest cleanup include_role: name: tempest-cleanup - when: run_tempest_cleanup is defined + when: run_tempest_cleanup is defined and run_tempest_cleanup | bool diff --git a/roles/tempest-cleanup/README.rst b/roles/tempest-cleanup/README.rst index 70719ca2cd..d1fad900a4 100644 --- a/roles/tempest-cleanup/README.rst +++ b/roles/tempest-cleanup/README.rst @@ -31,3 +31,31 @@ saved_state.json file. When true, tempest cleanup creates a report (./dry_run.json) of the resources that would be cleaned up if the role was ran with dry_run option set to false. + +.. zuul:rolevar:: run_tempest_fail_if_leaked_resources + :default: false + + When true, the role will fail if any leaked resources are detected. + The detection is done via dry_run.json file which if contains any resources, + some must have been leaked. This can be also used to verify that tempest + cleanup was successful. + + +Role usage +---------- + +The role can be also used for verification that tempest tests don't leak any +resources or to test that 'tempest cleanup' command deleted all leaked +resources as expected. +Either way the role needs to be run first with init_saved_state variable set +to true prior any tempest tests got executed. +Then, after tempest tests got executed this role needs to be run again with +role variables set according to the desired outcome: + +1. to verify that tempest tests don't leak any resources + run_tempest_dry_cleanup and run_tempest_fail_if_leaked_resources have to + be set to true. + +2. to check that 'tempest cleanup' command deleted all the leaked resources + run_tempest_cleanup and run_tempest_fail_if_leaked_resources have to be set + to true. diff --git a/roles/tempest-cleanup/defaults/main.yaml b/roles/tempest-cleanup/defaults/main.yaml index fc1948a13a..ce78bdbe38 100644 --- a/roles/tempest-cleanup/defaults/main.yaml +++ b/roles/tempest-cleanup/defaults/main.yaml @@ -1,3 +1,4 @@ devstack_base_dir: /opt/stack init_saved_state: false dry_run: false +run_tempest_fail_if_leaked_resources: false diff --git a/roles/tempest-cleanup/tasks/dry_run.yaml b/roles/tempest-cleanup/tasks/dry_run.yaml new file mode 100644 index 0000000000..46749ab5af --- /dev/null +++ b/roles/tempest-cleanup/tasks/dry_run.yaml @@ -0,0 +1,7 @@ +--- +- name: Run tempest cleanup dry-run + become: yes + become_user: tempest + command: tox -evenv-tempest -- tempest cleanup --dry-run --debug + args: + chdir: "{{ devstack_base_dir }}/tempest" diff --git a/roles/tempest-cleanup/tasks/dry_run_checker.py b/roles/tempest-cleanup/tasks/dry_run_checker.py new file mode 100644 index 0000000000..9cd9a8576a --- /dev/null +++ b/roles/tempest-cleanup/tasks/dry_run_checker.py @@ -0,0 +1,71 @@ +# Copyright 2020 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility for content checking of a given dry_run.json file. +""" + +import argparse +import json +import sys + + +def get_parser(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument('--is-empty', action="store_true", dest='is_empty', + default=False, + help="""Are values of a given dry_run.json empty?""") + parser.add_argument('--file', dest='file', default=None, metavar='PATH', + help="A path to a dry_run.json file.") + return parser + + +def parse_arguments(): + parser = get_parser() + args = parser.parse_args() + if not args.file: + sys.stderr.write('Path to a dry_run.json must be specified.\n') + sys.exit(1) + return args + + +def load_json(path): + """Load json content from file addressed by path.""" + try: + with open(path, 'rb') as json_file: + json_data = json.load(json_file) + except Exception as ex: + sys.exit(ex) + return json_data + + +def are_values_empty(dry_run_content): + """Return true if values of dry_run.json are empty.""" + for value in dry_run_content.values(): + if value: + return False + return True + + +def main(): + args = parse_arguments() + content = load_json(args.file) + if args.is_empty: + if not are_values_empty(content): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/roles/tempest-cleanup/tasks/main.yaml b/roles/tempest-cleanup/tasks/main.yaml index 5444afcc5f..c1d63f03eb 100644 --- a/roles/tempest-cleanup/tasks/main.yaml +++ b/roles/tempest-cleanup/tasks/main.yaml @@ -12,20 +12,35 @@ - when: dry_run block: - - name: Run tempest cleanup dry-run - become: yes - become_user: tempest - command: tox -evenv-tempest -- tempest cleanup --dry-run --debug - args: - chdir: "{{ devstack_base_dir }}/tempest" + - import_tasks: dry_run.yaml - name: Cat dry_run.json command: cat "{{ devstack_base_dir }}/tempest/dry_run.json" -- name: Run tempest cleanup - become: yes - become_user: tempest - command: tox -evenv-tempest -- tempest cleanup --debug - args: - chdir: "{{ devstack_base_dir }}/tempest" - when: not dry_run and not init_saved_state +- when: + - not dry_run + - not init_saved_state + block: + - name: Run tempest cleanup + become: yes + become_user: tempest + command: tox -evenv-tempest -- tempest cleanup --debug + args: + chdir: "{{ devstack_base_dir }}/tempest" + +- when: + - run_tempest_fail_if_leaked_resources + - not init_saved_state + block: + # let's run dry run again, if haven't already, to check no leftover + # resources were left behind after the cleanup in the previous task + - import_tasks: dry_run.yaml + when: not dry_run + + - name: Fail if any resources are leaked + become: yes + become_user: tempest + shell: | + python3 roles/tempest-cleanup/tasks/dry_run_checker.py --file {{ devstack_base_dir }}/tempest/dry_run.json --is-empty + args: + chdir: "{{ devstack_base_dir }}/tempest"