Added --ignore-test-failures CLI flag

Tox venv commands are expected to return a code that indicates
test cases final outcome:

 - 0 => any test vase has failed
 - 1 => some test case has failed
 - not in [0, 1] => any other error

This change handles the special case when some test cases have
failed. In any of below cases:
 - JENKINS_URL environment variable is defined on controller node
 - --ignore-test-failure flag is passed to IR plugin
then the plugin runs without treating it as an error.

This has been introduced because when running the plugin from
a Jenkins slave host, Jenkins itself is expected to check test
case have failed looking at produced Junit XML artifacts.

Change-Id: Ia7d7f736ae18a7561dced2271228d86ca658e892

infrared_plugin/jenkins.yaml

@@ -0,0 +1,7 @@
+---
+
+# Test cases failures are threated as success to allow Jenkinst to complete
+# yellow builds
+tox_expected_rcs:
+  - '{{ tox_succeeded_rc }}'
+  - '{{ tox_failed_rc }}'

infrared_plugin/main.yaml

@@ -43,6 +43,10 @@
         dest: '{{ playbook_dir }}/roles'
         state: link
 
+    - name: "include Jenkins settings"
+      include_vars: jenkins.yaml
+      when: "(lookup('env','JENKINS_URL') | length) > 0"
+
 
 - hosts:
     '{{ test_host | default(hostvars.localhost.test_host) | default("localhost") }}'

infrared_plugin/plugin.spec

@@ -171,6 +171,10 @@ subparsers:
             type: Value
             help: URL or path to upper contraints file
             ansible_variable: upper_constraints_file
+          ignore-test-failures:
+            type: Flag
+            help: Ignore test execution errors
+            ansible_variable: ignore_test_failures
 
       - title: Collect stage
         options:

roles/tobiko-tox/defaults/main.yaml

@@ -21,3 +21,10 @@ tox_constrain_env:
   TOX_REPORT_DIR: '{{ tox_report_dir }}'
   TOX_REPORT_NAME: '{{ tox_report_name }}'
   TOX_CONSTRAINTS_FILE: '{{ tox_constraints_file }}'
+
+tox_succeeded_rc: 0
+tox_failed_rc: 1
+
+tox_expected_rcs:
+  - '{{ tox_succeeded_rc }}'
+ignore_test_failures: no

roles/tobiko-tox/tasks/tox.yaml

@@ -20,6 +20,7 @@
       tox_description: '{{ tox_description }}'
       tox_dir: '{{ tox_dir }}'
       tox_environment: '{{ tox_environment | combine(tox_constrain_env) }}'
+      tox_expected_rcs: '{{ tox_expected_rcs }}'
 
 
 - name: "{{ tox_description }}"
@@ -29,27 +30,27 @@
   register:
     run_tox
   environment: '{{ tox_environment | combine(tox_constrain_env) }}'
+  failed_when: run_tox.rc != tox_succeeded_rc
   ignore_errors: yes
 
 
-- name: "show test cases results"
+- name: "show test cases output"
   debug: var=run_tox.stdout_lines
   when:
     - (run_tox.stdout_lines | length) > 0
 
 
-- name:
-  block:
-    - name: "show test cases errors"
-      debug: var=run_tox.stderr_lines
-      when:
-        - (run_tox.stderr_lines | length) > 0
-
-    - name: 'report test cases failure'
-      debug:
-        msg: 'test cases have failed'
-      failed_when: yes
-
+- name: "show test cases errors"
+  debug: var=run_tox.stderr_lines
   when:
-    - run_tox is defined
     - run_tox is failed
+    - (run_tox.stderr_lines | length) > 0
+
+
+- name: 'raise test cases failure'
+  debug:
+    msg: 'test cases have failed'
+  when: run_tox is failed
+  failed_when:
+    - not (ignore_test_failures | bool)
+    - run_tox.rc not in tox_expected_rcs

tools/run_tests.py

@@ -53,6 +53,7 @@ def main():
         succeeded = run_tests()
         if succeeded:
             LOG.info('SUCCEEDED')
+            sys.exit(0)
         else:
             LOG.info('FAILED')
             sys.exit(1)