Run all validations and fail after for CI role

The execution of validations in the CI role was checking the
result on each run and fail is something was not expected.
This patch failed the execution only after all the validations has been run.

This allow better analysis and debuging when more than one validations
are failing on a job.

Change-Id: I9b8f825a02f412475377cee34d88abe8431bc318

roles/validations/tasks/main.yaml

@@ -43,15 +43,37 @@
     validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
   when: not is_virtualenv.stat.exists
 
-- name: Run validations
+- name: Run positive validation tests
   include_tasks: run.yaml
   vars:
     name: "{{ item }}"
+    expected_rc: 0
   when:
     - run_validation|default(false)|bool
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
+- name: Fail if something went wrong
+  fail:
+    msg: "One or more Validations has failed, check the log results for more information."
+  when: result_failed | default(False) | bool
+
+- name: Run negative validation tests
+  include_tasks: run.yaml
+  vars:
+    name: "{{ item }}"
+    expected_rc: 1
+    negative: true
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"
+
+- name: Fail if something went wrong
+  fail:
+    msg: "One or more Validations has failed, check the log results for more information."
+  when: result_failed | default(False) | bool
+
 - name: List validations
   include_tasks: list.yaml
   vars:

roles/validations/tasks/run.yaml

@@ -1,77 +1,29 @@
 ---
-- name: Run validations - positive - these are supposed to pass
-  shell:
-    cmd: >-
-      {{ validation_command }} run --validation {{ name.key }}
-      {{ validation_dir }} {{ ansible_dir }}
-      --inventory {{ inventory }}
-      --output-log validation_{{ name.key }}_positive.log
-      {{ name.value.extra_args }}
-      {{ name.value.extra_env_args }}
-    executable: /bin/bash
-
-- name: Get Run results - positive - these are supposed to pass
+- name: Run validations
   block:
-    - name: Get run results
-      register: result
-      shell:
-        cmd: "cat validation_{{ name.key }}_positive.log"
-        executable: /bin/bash
-
-    - name: Get json data
+    - name: Set fact for extra args.
       set_fact:
-        jsondata: "{{ result.stdout | from_json }}"
-
-    - name: Get Validations Status
-      set_fact:
-        status: "{{ jsondata | json_query(jsonres) }}"
-      vars:
-        jsonres: 'results[*].Status'
-
-    - fail:
-        msg: "Validation failed with {{ validation_status }}: some of the validations have failed. {{ status }}"
-      when: validation_status != "PASSED"
-      loop: "{{ status }}"
-      loop_control:
-        loop_var: validation_status
-
-- name: Run validations - negative - these are not supposed to pass
-  shell:
-    cmd: >-
-      {{ validation_command }} run --validation {{ name.key }}
-      {{ validation_dir }} {{ ansible_dir }}
-      --inventory {{ inventory }}
-      --output-log validation_{{ name.key }}_negative.log
-      {{ name.value.negative_results.extra_args }}
-      {{ name.value.extra_env_args }}
-    executable: /bin/bash
-  when: "'negative_results' in name.value "
-  ignore_errors: true
-
-- name: Get Run results - negative - these are not supposed to pass
-  # This task should fail with return code != 0
-  # The validation is supplied with parameters that make it impossible to pass.
-  block:
-    - name: Get run results
-      register: result
+        execution_extra_args: "{{ name.value.negative_results.extra_args }}"
+      when:
+        - "'negative_results' in name.value "
+        - negative | default(False) | bool
+
+    - name: Execute Validations {{ name.key }}
+      ignore_errors: true
+      register: run_results
       shell:
-        cmd: "cat validation_{{ name.key }}_negative.log"
+        cmd: >-
+          {{ validation_command }} run --validation {{ name.key }}
+          {{ validation_dir }} {{ ansible_dir }}
+          --inventory {{ inventory }}
+          --output-log validation_{{ name.key }}_positive.log
+          {{ execution_extra_args | default(name.value.extra_args) }}
+          {{ name.value.extra_env_args }}
         executable: /bin/bash
+      when: (negative | default(False) | bool and 'negative_results' in name.value) or
+            (not negative | default(False) | bool)
 
-    - name: Get json data
-      set_fact:
-        jsondata: "{{ result.stdout | from_json }}"
-
-    - name: Get Validations Status
-      set_fact:
-        status: "{{ jsondata | json_query(jsonres) }}"
-      vars:
-        jsonres: 'results[*].Status'
-
-    - fail:
-        msg: "Validation passed with {{ validation_status }} when it shouldn't have: some of the validations have passed. {{ status }}"
-      when: validation_status == "PASSED"
-      loop: "{{ status }}"
-      loop_control:
-        loop_var: validation_status
-  when: "'negative_results' in name.value"
+    - name: set fact for run_results
+      ignore_errors: true
+      set_fact: result_failed=true
+      when: run_results.rc != expected_rc