Skip flaky test cases with IR plugin

Change-Id: I0320cdd00078f1c8a6acc19116fce88daed891c6

infrared_plugin/plugin.spec

@@ -187,6 +187,10 @@ subparsers:
             type: Value
             help: Python interpreter to be used for executing test cases
             ansible_variable: tox_python
+          pytest-addopts:
+            type: Value
+            help: Extra options to be passed to PyTest
+            ansible_variable: pytest_addopts
           run-tests-timeout:
             type: Value
             help: Timeout (in seconds) to interrupt test cases execution
@@ -207,6 +211,10 @@ subparsers:
             type: Flag
             help: Ignore test execution errors
             ansible_variable: ignore_test_failures
+          flaky:
+            type: Flag
+            help: Ignore flaky test cases
+            ansible_variable: test_flaky
 
       - title: Cleanup stage
         options:

roles/tobiko-ensure-python3/vars/py3-MacOSX.yaml

@@ -1,12 +1,10 @@
 ---
 
 python_exe_dirs:
+  - ~/Library/Python/3.10/bin
   - ~/Library/Python/3.8/bin
-  - ~/Library/Python/3.7/bin
-  - ~/Library/Python/3.6/bin
+  - /Library/Frameworks/Python.framework/Versions/3.10/bin
   - /Library/Frameworks/Python.framework/Versions/3.8/bin
-  - /Library/Frameworks/Python.framework/Versions/3.7/bin
-  - /Library/Frameworks/Python.framework/Versions/3.6/bin
   - /usr/local/bin
   - /usr/bin
   - /bin

roles/tobiko-run/vars/test-workflow-functional.yaml

@@ -3,3 +3,4 @@
 test_workflow_steps:
   - tox_description: 'run functional test cases'
     tox_envlist: functional
+    test_flaky: true

roles/tobiko-run/vars/test-workflow-gate.yaml

@@ -4,37 +4,44 @@ test_workflow_steps:
   - tox_description: 'run unit test cases'
     tox_envlist: py3
     tox_step_name: unit
+    test_flaky: true
 
   - tox_description: 'run functional test cases'
     tox_envlist: functional
     tox_step_name: functional
+    test_flaky: true
 
   - tox_description: 'create workload resources'
     tox_envlist: scenario
     tox_step_name: create_resources
     tox_environment:
       TOBIKO_PREVENT_CREATE: no
+    test_flaky: true
 
   - tox_description: 'run sanity test cases before disruptive test cases'
     tox_envlist: sanity
     tox_step_name: before_faults
     tox_environment:
       TOBIKO_PREVENT_CREATE: no
+    test_flaky: true
 
   - tox_description: 'run disruptive test cases'
     tox_envlist: faults
     tox_step_name: faults
     tox_environment:
       TOBIKO_PREVENT_CREATE: no
+    test_flaky: true
 
   - tox_description: 'run sanity test cases after disruptive test cases'
     tox_envlist: sanity
     tox_step_name: after_faults
     tox_environment:
       TOBIKO_PREVENT_CREATE: no
+    test_flaky: true
 
   - tox_description: 'verify workload resources'
     tox_envlist: scenario
     tox_step_name: verify_resources
     tox_environment:
       TOBIKO_PREVENT_CREATE: yes
+    test_flaky: true

roles/tobiko-run/vars/test-workflow-unit.yaml

@@ -3,13 +3,28 @@ test_workflow_steps:
     tox_envlist: py36
     python_version: '3.6'
     bindep_profile: test py36
+    test_flaky: true
 
   - tox_description: 'run unit tests using Python 3.7'
     tox_envlist: py37
     python_version: '3.7'
     bindep_profile: test py37
+    test_flaky: true
 
   - tox_description: 'run unit tests using Python 3.8'
     tox_envlist: py38
     python_version: '3.8'
     bindep_profile: test py38
+    test_flaky: true
+
+  - tox_description: 'run unit tests using Python 3.9'
+    tox_envlist: py39
+    python_version: '3.9'
+    bindep_profile: test py39
+    test_flaky: true
+
+  - tox_description: 'run unit tests using Python 3.10'
+    tox_envlist: py310
+    python_version: '3.10'
+    bindep_profile: test py310
+    test_flaky: true

roles/tobiko-tox/defaults/main.yaml

@@ -20,6 +20,8 @@ tox_run_tests_timeout: 18000  # 5 hours
 tox_constraints: ''
 
 tox_constrain_env:
+  PYTEST_ADDOPTS:
+    "{% if not test_flaky %}-m 'not flaky' {% endif %}{{ pytest_addopts }}"
   TOX_REPORT_DIR: '{{ tox_report_dir }}'
   TOX_REPORT_NAME: '{{ tox_report_name }}'
   TOX_CONSTRAINTS: '{{ tox_constraints }}'
@@ -31,3 +33,6 @@ tox_failed_rc: 1
 tox_expected_rcs:
   - '{{ tox_succeeded_rc }}'
 ignore_test_failures: no
+
+pytest_addopts: ''
+test_flaky: false