Better tempest blacklist management

This patch set the information regarding how the tempest_test_blacklist
variable should be configured. From now, each test in the blacklist
should have a launchpad and/or bugzilla tighted on it in order to
trackdown the progress of the test, as well as use other tools to
automate the verification of the status of those bugs.

Change-Id: I36c41d97ca778d892c1655731cca9ac0112851c1

defaults/main.yml

@@ -106,12 +106,17 @@ tempest_test_whitelist:
   - "{{ (tempest_service_available_nova | bool) | ternary('tempest.scenario.test_server_basic_ops', '') }}"
   - "{{ (tempest_service_available_swift | bool) | ternary('tempest.scenario.test_object_storage_basic_ops', '') }}"
   - "{{ (tempest_volume_multi_backend_enabled | bool) | ternary('tempest.api.volume.admin.test_multi_backend', '') }}"
-# TODO(evrardjp): Bring this back when upstream is fixed
-# (revert of https://github.com/openstack/cinder/commit/737c50b4ea54e2e63d6e3fe8a73d22b393df4205)
-#  - "{{ (tempest_volume_backup_enabled | bool) | ternary('tempest.api.volume.admin.test_volumes_backup', '') }}"
+  # TODO(evrardjp): Bring this back when upstream is fixed
+  # (revert of https://github.com/openstack/cinder/commit/737c50b4ea54e2e63d6e3fe8a73d22b393df4205)
+  #  - "{{ (tempest_volume_backup_enabled | bool) | ternary('tempest.api.volume.admin.test_volumes_backup', '') }}"
 
-# Tests to NOT execute:
-# This sets up a list of tests to skip, which can even include those included in the whitelist.
+# Tests being skipped by os_tempest
+# Example:
+# tempest_test_blacklist:
+#   - test: tempest.scenario.test.minimum_basic
+#     reason: This test is failing
+#     lp: 'https://bugs.launchpad.net/openstack-ansible/+bug/123456'
+#     bz: 'https://bugzilla.redhat.com/show_bug.cgi?id=123456'
 tempest_test_blacklist: []
 
 # Toggle fatal deprecations

releasenotes/notes/blacklist-format-26c6097cf4e813c9.yaml

@@ -0,0 +1,12 @@
+---
+upgrade: >
+    Data structure for ``tempest_test_blacklist`` has been updated to
+    add launchpad and/or bugzilla linked with the test being skipped.
+features:
+  - |
+    Add the launchpad and bugzilla keys in tempest_test_blacklist ansible
+    variable.
+    Developers must have a way to trackdown why a test was inserted in the
+    skiplist, and one of the ways is through bugs. This feature add the
+    information regarding it in the list of skipped tests on os_tempest
+

tasks/tempest_post_install.yml

@@ -99,13 +99,17 @@
       {%   endif %}
       {% endfor %}
     dest: "{{ tempest_test_whitelist_file_path }}"
+  when:
+    - tempest_test_whitelist | length > 0
 
+# Tests to NOT execute:
+# This sets up a list of tests to skip, which can even include those included in the whitelist.
 - name: Generate tempest test blacklist
   copy:
     content: |
-      {% for item in tempest_test_blacklist | unique | sort %}
-      {%   if item != '' %}
-      {{ item }}
+      {% for item in tempest_test_blacklist %}
+      {% if item.test != '' %}
+      {{ item.test }}
       {%   endif %}
       {% endfor %}
     dest: "{{ tempest_test_blacklist_file_path }}"