From 2976535577273a4d423c4afe7228ae7228f5e2fb Mon Sep 17 00:00:00 2001
From: matbu <mat.bultel@gmail.com>
Date: Thu, 25 Nov 2021 13:33:32 +0100
Subject: [PATCH] Run all validations and fail after for CI role

The execution of validations in the CI role was checking the
result on each run and fail is something was not expected.
This patch failed the execution only after all the validations has been run.

This allow better analysis and debuging when more than one validations
are failing on a job.

Change-Id: I9b8f825a02f412475377cee34d88abe8431bc318
---
 roles/validations/tasks/main.yaml | 24 +++++++-
 roles/validations/tasks/run.yaml  | 96 ++++++++-----------------------
 2 files changed, 47 insertions(+), 73 deletions(-)

diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml
index 7a37b55..9febee5 100644
--- a/roles/validations/tasks/main.yaml
+++ b/roles/validations/tasks/main.yaml
@@ -43,15 +43,37 @@
     validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
   when: not is_virtualenv.stat.exists
 
-- name: Run validations
+- name: Run positive validation tests
   include_tasks: run.yaml
   vars:
     name: "{{ item }}"
+    expected_rc: 0
   when:
     - run_validation|default(false)|bool
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
+- name: Fail if something went wrong
+  fail:
+    msg: "One or more Validations has failed, check the log results for more information."
+  when: result_failed | default(False) | bool
+
+- name: Run negative validation tests
+  include_tasks: run.yaml
+  vars:
+    name: "{{ item }}"
+    expected_rc: 1
+    negative: true
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"
+
+- name: Fail if something went wrong
+  fail:
+    msg: "One or more Validations has failed, check the log results for more information."
+  when: result_failed | default(False) | bool
+
 - name: List validations
   include_tasks: list.yaml
   vars:
diff --git a/roles/validations/tasks/run.yaml b/roles/validations/tasks/run.yaml
index 2d7e38a..84770aa 100644
--- a/roles/validations/tasks/run.yaml
+++ b/roles/validations/tasks/run.yaml
@@ -1,77 +1,29 @@
 ---
-- name: Run validations - positive - these are supposed to pass
-  shell:
-    cmd: >-
-      {{ validation_command }} run --validation {{ name.key }}
-      {{ validation_dir }} {{ ansible_dir }}
-      --inventory {{ inventory }}
-      --output-log validation_{{ name.key }}_positive.log
-      {{ name.value.extra_args }}
-      {{ name.value.extra_env_args }}
-    executable: /bin/bash
-
-- name: Get Run results - positive - these are supposed to pass
+- name: Run validations
   block:
-    - name: Get run results
-      register: result
+    - name: Set fact for extra args.
+      set_fact:
+        execution_extra_args: "{{ name.value.negative_results.extra_args }}"
+      when:
+        - "'negative_results' in name.value "
+        - negative | default(False) | bool
+
+    - name: Execute Validations {{ name.key }}
+      ignore_errors: true
+      register: run_results
       shell:
-        cmd: "cat validation_{{ name.key }}_positive.log"
+        cmd: >-
+          {{ validation_command }} run --validation {{ name.key }}
+          {{ validation_dir }} {{ ansible_dir }}
+          --inventory {{ inventory }}
+          --output-log validation_{{ name.key }}_positive.log
+          {{ execution_extra_args | default(name.value.extra_args) }}
+          {{ name.value.extra_env_args }}
         executable: /bin/bash
+      when: (negative | default(False) | bool and 'negative_results' in name.value) or
+            (not negative | default(False) | bool)
 
-    - name: Get json data
-      set_fact:
-        jsondata: "{{ result.stdout | from_json }}"
-
-    - name: Get Validations Status
-      set_fact:
-        status: "{{ jsondata | json_query(jsonres) }}"
-      vars:
-        jsonres: 'results[*].Status'
-
-    - fail:
-        msg: "Validation failed with {{ validation_status }}: some of the validations have failed. {{ status }}"
-      when: validation_status != "PASSED"
-      loop: "{{ status }}"
-      loop_control:
-        loop_var: validation_status
-
-- name: Run validations - negative - these are not supposed to pass
-  shell:
-    cmd: >-
-      {{ validation_command }} run --validation {{ name.key }}
-      {{ validation_dir }} {{ ansible_dir }}
-      --inventory {{ inventory }}
-      --output-log validation_{{ name.key }}_negative.log
-      {{ name.value.negative_results.extra_args }}
-      {{ name.value.extra_env_args }}
-    executable: /bin/bash
-  when: "'negative_results' in name.value "
-  ignore_errors: true
-
-- name: Get Run results - negative - these are not supposed to pass
-  # This task should fail with return code != 0
-  # The validation is supplied with parameters that make it impossible to pass.
-  block:
-    - name: Get run results
-      register: result
-      shell:
-        cmd: "cat validation_{{ name.key }}_negative.log"
-        executable: /bin/bash
-
-    - name: Get json data
-      set_fact:
-        jsondata: "{{ result.stdout | from_json }}"
-
-    - name: Get Validations Status
-      set_fact:
-        status: "{{ jsondata | json_query(jsonres) }}"
-      vars:
-        jsonres: 'results[*].Status'
-
-    - fail:
-        msg: "Validation passed with {{ validation_status }} when it shouldn't have: some of the validations have passed. {{ status }}"
-      when: validation_status == "PASSED"
-      loop: "{{ status }}"
-      loop_control:
-        loop_var: validation_status
-  when: "'negative_results' in name.value"
+    - name: set fact for run_results
+      ignore_errors: true
+      set_fact: result_failed=true
+      when: run_results.rc != expected_rc