Allow include/exclude lists to be defined in many variables

Previously the os_tempest role had a single variable for
defining the include and exclude lists in order to select the
tempest tests to run.

This works for simple scenarios, where a single service
is deployed and the tests for that service are enabled through
the necessary variables set in user_variables. This approach is
used in openstack-ansible CI / AIO.

More complicated scenarios such as magnum+barbican+octavia, would
create several user_variables files each with conflicting settings
for the test settings. It is possible in this scenario for there to
be no valid tempest tests to run and tempest to fail immediately.

This patch adds the possibility to have many variables defining
the include/exclude lists which have names using a common prefix.
Any variable names matching the prefix are gathered and combined
with the original role default to make extending the test lists
easy to do in an incremental/distibuted way in the ansible variables
instead of having to maintain a single point defining all necessary
tests.

Change-Id: Ie3a9a7be849171af042567ba8a152e5df5d2cb53

defaults/main.yml

@@ -168,14 +168,14 @@ tempest_workspace: "{{ ansible_facts['env']['HOME'] }}/workspace"
 tempest_includelist_file_path: "{{ tempest_workspace }}/etc/tempest_includelist.txt"
 tempest_excludelist_file_path: "{{ tempest_workspace }}/etc/tempest_excludelist.txt"
 
+# Variable prefixes that will be dynamically gathered to create the include/exclude list
+tempest_test_search_includelist_pattern: 'tempest_test_includelist_'
+tempest_test_search_excludelist_pattern: 'tempest_test_excludelist_'
+
 # Tests to execute:
 # This sets up a list of tests to execute based on what's deployed in the environment.
 # The list gets added to the includelist which tempest executes.
-tempest_test_includelist: "{{ tempest_test_default_includelist }}"
-tempest_test_default_includelist:
-  - "smoke"
-  - "{{ (tempest_service_available_ceilometer | bool) | ternary('tempest.api.telemetry', '') }}"
-  - "{{ (tempest_service_available_heat | bool) | ternary('tempest.api.orchestration.stacks.test_non_empty_stack', '') }}"
+tempest_test_includelist: []
 
 # Extra test to be executed
 tempest_test_extra_test: []

releasenotes/notes/gather-include-exclude-lists-91007886c06ebb74.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    It is now possible to use multiple variables with a specific prefix to
+    define the whole contents of the tempest test include/exclude lists.
+    Any variable from host/group or ansible extra-vars whose name is prefixed
+    with the value in the os_tempest role default `tempest_test_search_includelist_pattern`
+    or `tempest_test_search_excludelist_pattern` will be combined with the
+    existing `tempest_test_includelist` or `tempest_test_excludelist` variables
+    into a single include/exclude list.

tasks/tempest_post_install.yml

@@ -158,7 +158,7 @@
 - name: Generate tempest test include list
   copy:
     content: |
-      {% for item in (tempest_test_includelist + tempest_test_extra_test) | unique | sort %}
+      {% for item in (_tempest_test_includelist + tempest_test_extra_test) | unique | sort %}
       {%   if item %}
       {{ item }}
       {%   endif %}
@@ -166,14 +166,14 @@
     dest: "{{ tempest_includelist_file_path }}"
     mode: "0644"
   when:
-    - tempest_test_includelist | length > 0
+    - _tempest_test_includelist | length > 0
 
 # Tests to NOT execute:
 # This sets up a list of tests to skip, which can even include those included in the includelist.
 - name: Generate tempest test exclude list
   copy:
     content: |
-      {% for item in tempest_test_excludelist %}
+      {% for item in _tempest_test_excludelist %}
       {% if item.test is defined %}
       {{ item.test }}
       {% else %}
@@ -183,7 +183,7 @@
     dest: "{{ tempest_excludelist_file_path }}"
     mode: "0644"
   when:
-    - tempest_test_excludelist | length > 0
+    - _tempest_test_excludelist | length > 0
 
 - name: Drop test_accounts_file
   copy:

vars/main.yml

@@ -133,3 +133,13 @@ _tempest_plugins:
     repo: "{{ tempest_plugin_zun_git_repo }}"
     branch: "{{ tempest_plugin_zun_git_install_branch }}"
     install: "{{ tempest_service_available_zun | bool }}"
+
+# gather include/exclude lists from any variables starting with the defined search pattern
+# allows many different ansible vars to be combined easily to make a single include/exclude list
+_tempest_test_gathered_includelist: "{{ query('vars', *query('varnames', '^' ~ tempest_test_search_includelist_pattern)) | flatten(levels=1) }}"
+_tempest_test_gathered_excludelist: "{{ query('vars', *query('varnames', '^' ~ tempest_test_search_excludelist_pattern)) | flatten(levels=1) }}"
+
+# maintain backward compatibility by combining the original role default
+# with any dynamically gathered variables
+_tempest_test_includelist: "{{ tempest_test_includelist | union(_tempest_test_gathered_includelist) }}"
+_tempest_test_excludelist: "{{ tempest_test_excludelist | union(_tempest_test_gathered_excludelist) }}"