From 4c3e0ba4e444afba3477cbfdecde263f44ff7a61 Mon Sep 17 00:00:00 2001
From: Arx Cruz <arxcruz@redhat.com>
Date: Mon, 3 Dec 2018 16:26:29 +0100
Subject: [PATCH] Better tempest blacklist management

This patch set the information regarding how the tempest_test_blacklist
variable should be configured. From now, each test in the blacklist
should have a launchpad and/or bugzilla tighted on it in order to
trackdown the progress of the test, as well as use other tools to
automate the verification of the status of those bugs.

Change-Id: I36c41d97ca778d892c1655731cca9ac0112851c1
---
 defaults/main.yml                                 | 15 ++++++++++-----
 .../notes/blacklist-format-26c6097cf4e813c9.yaml  | 12 ++++++++++++
 tasks/tempest_post_install.yml                    | 10 +++++++---
 3 files changed, 29 insertions(+), 8 deletions(-)
 create mode 100644 releasenotes/notes/blacklist-format-26c6097cf4e813c9.yaml

diff --git a/defaults/main.yml b/defaults/main.yml
index cf16eb9d..72c41879 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -106,12 +106,17 @@ tempest_test_whitelist:
   - "{{ (tempest_service_available_nova | bool) | ternary('tempest.scenario.test_server_basic_ops', '') }}"
   - "{{ (tempest_service_available_swift | bool) | ternary('tempest.scenario.test_object_storage_basic_ops', '') }}"
   - "{{ (tempest_volume_multi_backend_enabled | bool) | ternary('tempest.api.volume.admin.test_multi_backend', '') }}"
-# TODO(evrardjp): Bring this back when upstream is fixed
-# (revert of https://github.com/openstack/cinder/commit/737c50b4ea54e2e63d6e3fe8a73d22b393df4205)
-#  - "{{ (tempest_volume_backup_enabled | bool) | ternary('tempest.api.volume.admin.test_volumes_backup', '') }}"
+  # TODO(evrardjp): Bring this back when upstream is fixed
+  # (revert of https://github.com/openstack/cinder/commit/737c50b4ea54e2e63d6e3fe8a73d22b393df4205)
+  #  - "{{ (tempest_volume_backup_enabled | bool) | ternary('tempest.api.volume.admin.test_volumes_backup', '') }}"
 
-# Tests to NOT execute:
-# This sets up a list of tests to skip, which can even include those included in the whitelist.
+# Tests being skipped by os_tempest
+# Example:
+# tempest_test_blacklist:
+#   - test: tempest.scenario.test.minimum_basic
+#     reason: This test is failing
+#     lp: 'https://bugs.launchpad.net/openstack-ansible/+bug/123456'
+#     bz: 'https://bugzilla.redhat.com/show_bug.cgi?id=123456'
 tempest_test_blacklist: []
 
 # Toggle fatal deprecations
diff --git a/releasenotes/notes/blacklist-format-26c6097cf4e813c9.yaml b/releasenotes/notes/blacklist-format-26c6097cf4e813c9.yaml
new file mode 100644
index 00000000..e4c94a8c
--- /dev/null
+++ b/releasenotes/notes/blacklist-format-26c6097cf4e813c9.yaml
@@ -0,0 +1,12 @@
+---
+upgrade: >
+    Data structure for ``tempest_test_blacklist`` has been updated to
+    add launchpad and/or bugzilla linked with the test being skipped.
+features:
+  - |
+    Add the launchpad and bugzilla keys in tempest_test_blacklist ansible
+    variable.
+    Developers must have a way to trackdown why a test was inserted in the
+    skiplist, and one of the ways is through bugs. This feature add the
+    information regarding it in the list of skipped tests on os_tempest
+
diff --git a/tasks/tempest_post_install.yml b/tasks/tempest_post_install.yml
index 7ae962d5..9dc64066 100644
--- a/tasks/tempest_post_install.yml
+++ b/tasks/tempest_post_install.yml
@@ -99,13 +99,17 @@
       {%   endif %}
       {% endfor %}
     dest: "{{ tempest_test_whitelist_file_path }}"
+  when:
+    - tempest_test_whitelist | length > 0
 
+# Tests to NOT execute:
+# This sets up a list of tests to skip, which can even include those included in the whitelist.
 - name: Generate tempest test blacklist
   copy:
     content: |
-      {% for item in tempest_test_blacklist | unique | sort %}
-      {%   if item != '' %}
-      {{ item }}
+      {% for item in tempest_test_blacklist %}
+      {% if item.test != '' %}
+      {{ item.test }}
       {%   endif %}
       {% endfor %}
     dest: "{{ tempest_test_blacklist_file_path }}"