diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index b49ec0d71..e48d07889 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -187,6 +187,10 @@ subparsers: type: Value help: Python interpreter to be used for executing test cases ansible_variable: tox_python + pytest-addopts: + type: Value + help: Extra options to be passed to PyTest + ansible_variable: pytest_addopts run-tests-timeout: type: Value help: Timeout (in seconds) to interrupt test cases execution @@ -207,6 +211,10 @@ subparsers: type: Flag help: Ignore test execution errors ansible_variable: ignore_test_failures + flaky: + type: Flag + help: Ignore flaky test cases + ansible_variable: test_flaky - title: Cleanup stage options: diff --git a/roles/tobiko-ensure-python3/vars/py3-MacOSX.yaml b/roles/tobiko-ensure-python3/vars/py3-MacOSX.yaml index 117dc8761..a0bec3053 100644 --- a/roles/tobiko-ensure-python3/vars/py3-MacOSX.yaml +++ b/roles/tobiko-ensure-python3/vars/py3-MacOSX.yaml @@ -1,12 +1,10 @@ --- python_exe_dirs: + - ~/Library/Python/3.10/bin - ~/Library/Python/3.8/bin - - ~/Library/Python/3.7/bin - - ~/Library/Python/3.6/bin + - /Library/Frameworks/Python.framework/Versions/3.10/bin - /Library/Frameworks/Python.framework/Versions/3.8/bin - - /Library/Frameworks/Python.framework/Versions/3.7/bin - - /Library/Frameworks/Python.framework/Versions/3.6/bin - /usr/local/bin - /usr/bin - /bin diff --git a/roles/tobiko-run/vars/test-workflow-functional.yaml b/roles/tobiko-run/vars/test-workflow-functional.yaml index d70d28606..bcc2e5292 100644 --- a/roles/tobiko-run/vars/test-workflow-functional.yaml +++ b/roles/tobiko-run/vars/test-workflow-functional.yaml @@ -3,3 +3,4 @@ test_workflow_steps: - tox_description: 'run functional test cases' tox_envlist: functional + test_flaky: true diff --git a/roles/tobiko-run/vars/test-workflow-gate.yaml b/roles/tobiko-run/vars/test-workflow-gate.yaml index 8d3d0ea57..552a356bb 100644 --- a/roles/tobiko-run/vars/test-workflow-gate.yaml +++ b/roles/tobiko-run/vars/test-workflow-gate.yaml @@ -4,37 +4,44 @@ test_workflow_steps: - tox_description: 'run unit test cases' tox_envlist: py3 tox_step_name: unit + test_flaky: true - tox_description: 'run functional test cases' tox_envlist: functional tox_step_name: functional + test_flaky: true - tox_description: 'create workload resources' tox_envlist: scenario tox_step_name: create_resources tox_environment: TOBIKO_PREVENT_CREATE: no + test_flaky: true - tox_description: 'run sanity test cases before disruptive test cases' tox_envlist: sanity tox_step_name: before_faults tox_environment: TOBIKO_PREVENT_CREATE: no + test_flaky: true - tox_description: 'run disruptive test cases' tox_envlist: faults tox_step_name: faults tox_environment: TOBIKO_PREVENT_CREATE: no + test_flaky: true - tox_description: 'run sanity test cases after disruptive test cases' tox_envlist: sanity tox_step_name: after_faults tox_environment: TOBIKO_PREVENT_CREATE: no + test_flaky: true - tox_description: 'verify workload resources' tox_envlist: scenario tox_step_name: verify_resources tox_environment: TOBIKO_PREVENT_CREATE: yes + test_flaky: true diff --git a/roles/tobiko-run/vars/test-workflow-unit.yaml b/roles/tobiko-run/vars/test-workflow-unit.yaml index 7b8ebe607..0a0c89223 100644 --- a/roles/tobiko-run/vars/test-workflow-unit.yaml +++ b/roles/tobiko-run/vars/test-workflow-unit.yaml @@ -3,13 +3,28 @@ test_workflow_steps: tox_envlist: py36 python_version: '3.6' bindep_profile: test py36 + test_flaky: true - tox_description: 'run unit tests using Python 3.7' tox_envlist: py37 python_version: '3.7' bindep_profile: test py37 + test_flaky: true - tox_description: 'run unit tests using Python 3.8' tox_envlist: py38 python_version: '3.8' bindep_profile: test py38 + test_flaky: true + + - tox_description: 'run unit tests using Python 3.9' + tox_envlist: py39 + python_version: '3.9' + bindep_profile: test py39 + test_flaky: true + + - tox_description: 'run unit tests using Python 3.10' + tox_envlist: py310 + python_version: '3.10' + bindep_profile: test py310 + test_flaky: true diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index f11fdb092..a65a02fde 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -20,6 +20,8 @@ tox_run_tests_timeout: 18000 # 5 hours tox_constraints: '' tox_constrain_env: + PYTEST_ADDOPTS: + "{% if not test_flaky %}-m 'not flaky' {% endif %}{{ pytest_addopts }}" TOX_REPORT_DIR: '{{ tox_report_dir }}' TOX_REPORT_NAME: '{{ tox_report_name }}' TOX_CONSTRAINTS: '{{ tox_constraints }}' @@ -31,3 +33,6 @@ tox_failed_rc: 1 tox_expected_rcs: - '{{ tox_succeeded_rc }}' ignore_test_failures: no + +pytest_addopts: '' +test_flaky: false