Functional test for running validations from file

Functional tests for a change Ifc6c28003c4c2c5f3dd6198e650f9713a02dc82d

In order to use the 'validation_dir' and 'ansible_dir' parameters in the YAML file when executing the 'file' command, the original parameters 'validation_dir' and 'ansible_dir'  are changed to 'validation_dir_arg' and 'ansible_dir_arg' when using as a argument in the 'run' command.

Related-to: rhbz#2122209

Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
Change-Id: I56d974814987ef7a1cc105d4efe3123dd97f6b3e

roles/validations/defaults/main.yaml

@@ -23,3 +23,5 @@ cli_command: "validation"
 run_validation: true
 execute_full_vf_catalogue: "{{ job.execute_full_vf_catalogue|default(false)|bool }}"
 vf_catalogue_overrides: "{{ ansible_user_dir }}/catalog_vars_override.yaml"
+files_to_run_dest: "{{ ansible_user_dir }}"
+files_test_failure: false

roles/validations/tasks/execute_full_catalogue.yaml

@@ -7,7 +7,7 @@
   shell:
     cmd: >-
       {{ validation_command }} run --validation {{ validation_catalogue.full_list | join(",") }}
-      {{ validation_dir }} {{ ansible_dir }}
+      {{ validation_dir_arg }} {{ ansible_dir_arg }}
       --inventory {{ inventory }}
       --output-log validation_catalogue_execution.log
       {{ validation_catalogue.extra_args }}

roles/validations/tasks/file.yaml

@@ -0,0 +1,118 @@
+---
+- name: Run validations from the YAML file
+  vars:
+    file_wrong_path: foo/bar.yaml
+    junitxml_path: /home/stack/logs
+    junitxml_wrong_path: /foo/bar
+    wrong_path_string: must be properly formatted
+    wrong_directory_string: No such file or directory
+    junitxml_missing_string: junitxml output disabled
+    validation_failed_string: have failed
+    validation_not_found_string: Following validations were not found
+    no_validation_run_string: No validation has been run
+    expected_result: 1
+  block:
+    - debug:
+        msg: "{{ item }}"
+      loop: "{{ files }}"
+
+    - name: Execute the file command
+      block:
+        - name: Passed validation test
+          ignore_errors: true
+          register: run_results
+          shell:
+            cmd: >-
+              {{ validation_command }} file {{ files[0] }}
+            executable: /bin/bash
+
+        - set_fact:
+            files_test_failure: true
+          when: "{{ run_results.rc }} == 1"
+
+        - name: Failed validation test
+          ignore_errors: true
+          register: run_results
+          shell:
+            cmd: >-
+              {{ validation_command }} file {{ files[3] }}
+            executable: /bin/bash
+
+        - set_fact:
+            files_test_failure: true
+          when: run_results.rc != expected_result
+
+        - name: Run with no executed validation
+          ignore_errors: true
+          register: run_results
+          shell:
+            cmd: >-
+              {{ validation_command }} file {{ files[1] }}
+            executable: /bin/bash
+
+        - set_fact:
+            files_test_failure: true
+          when:
+            - run_results.rc != expected_result
+
+        - name: Run with non-existent validation
+          ignore_errors: true
+          register: run_results
+          shell:
+            cmd: >-
+              {{ validation_command }} file {{ files[2] }}
+            executable: /bin/bash
+
+        - set_fact:
+            files_test_failure: true
+          when:
+            - validation_not_found_string not in run_results.stdout
+            - validation_not_found_string not in run_results.stderr
+
+    - name: Execute the file command with wrong path
+      ignore_errors: true
+      register: run_results
+      shell:
+        cmd: >-
+          {{ validation_command }} file {{ file_wrong_path }}
+        executable: /bin/bash
+
+    - set_fact:
+        files_test_failure: true
+      when:
+        - wrong_path_string not in run_results.stdout
+        - wrong_path_string not in run_results.stderr
+
+    - name: Execute the file command with junitxml output
+      ignore_errors: true
+      register: run_results
+      shell:
+        cmd: >-
+          {{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_path }}
+        executable: /bin/bash
+
+    - set_fact:
+        files_test_failure: true
+      when:
+        - wrong_directory_string in run_results.stdout
+        - junitxml_missing_string not in run_results.stdout
+        - wrong_directory_string in run_results.stderr
+        - junitxml_missing_string not in run_results.stderr
+        - "{{ run_results.rc }} == 1"
+
+    - name: Execute the file command with wrong Junitxml path
+      ignore_errors: true
+      register: run_results
+      shell:
+        cmd: >-
+          {{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_wrong_path }}
+        executable: /bin/bash
+
+    - set_fact:
+        files_test_failure: true
+      when:
+        - wrong_directory_string in run_results.stdout
+        - junitxml_missing_string not in run_results.stdout
+        - wrong_directory_string in run_results.stderr
+        - junitxml_missing_string not in run_results.stderr
+        - "{{ run_results.rc }} == 1"

roles/validations/tasks/list.yaml

@@ -1,19 +1,19 @@
 ---
 - name: List Validations - all - to file
   shell:
-    cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1"
+    cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f json > {{ val_working_dir }}/list.log 2>&1"
     executable: /bin/bash
   when: val_format == "json"
 
 - name: List Validations - all - to stdout - {{ val_format }}
   shell:
-    cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}"
+    cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f {{ val_format }}"
     executable: /bin/bash
 
 # Metadata dependent list output
 - name: List Validations - group - to stdout - {{ val_format }}
   shell:
-    cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}"
+    cmd: "{{ validation_command }} list {{ validation_dir_arg }} --group {{ val_group }} -f {{ val_format }}"
     executable: /bin/bash
   loop: "{{ validation_metadata.group }}"
   loop_control:
@@ -21,7 +21,7 @@
 
 - name: " List Validations - category - to stdout - {{ val_format }} "
   shell:
-    cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}"
+    cmd: "{{ validation_command }} list {{ validation_dir_arg }} --category {{ val_category }} -f {{ val_format }}"
     executable: /bin/bash
   loop: "{{ validation_metadata.category }}"
   loop_control:
@@ -29,7 +29,7 @@
 
 - name: "List Validations - product - to stdout - {{ val_format }}"
   shell:
-    cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}"
+    cmd: "{{ validation_command }} list {{ validation_dir_arg }} --product {{ val_product }} -f {{ val_format }}"
     executable: /bin/bash
   loop: "{{ validation_metadata.product }}"
   loop_control:

roles/validations/tasks/main.yaml

@@ -39,12 +39,22 @@
 
 - name: Set Validation directory if virtualenv exists
   set_fact:
-    validation_dir: "--validation-dir {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks"
+    validation_dir: "{{ zuul_work_virtualenv }}/share/ansible/validation-playbooks"
   when: is_virtualenv.stat.exists
 
-- name: Set Ansible base directory if virtualenv exists
+- name: Set Validation directory argument if virtualenv exists
   set_fact:
-    ansible_dir: "--ansible-base-dir {{ zuul_work_virtualenv }}/share/ansible/"
+    validation_dir_arg: "--validation-dir {{ validation_dir }}"
+  when: is_virtualenv.stat.exists
+
+- name: Set Ansible base directory path if virtualenv exists
+  set_fact:
+    ansible_dir: "{{ zuul_work_virtualenv }}/share/ansible/"
+  when: is_virtualenv.stat.exists
+
+- name: Set Ansible base directory argument if virtualenv exists
+  set_fact:
+    ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}"
   when: is_virtualenv.stat.exists
 
 - name: Set commmand without virtualenv
@@ -56,7 +66,22 @@
 
 - name: Set validation dir without virtualenv
   set_fact:
-    validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
+    validation_dir: "/usr/share/ansible/validation-playbooks"
+  when: not is_virtualenv.stat.exists
+
+- name: Set validation dir argument without virtualenv
+  set_fact:
+    validation_dir_arg: "--validation-dir {{ validation_dir }}"
+  when: not is_virtualenv.stat.exists
+
+- name: Set Ansible base directory path withnout virtualenv
+  set_fact:
+    ansible_dir: "/usr/share/ansible/"
+  when: not is_virtualenv.stat.exists
+
+- name: Set Ansible base directory argument withnout virtualenv exists
+  set_fact:
+    ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}"
   when: not is_virtualenv.stat.exists
 
 - name: Set a valid inventory
@@ -167,3 +192,39 @@
 - name: Execute entire validations catalogue
   include_tasks: execute_full_catalogue.yaml
   when: execute_full_vf_catalogue
+
+- debug:
+    msg: "{{ item }}"
+  loop: "{{ test_arguments_run_from_file }}"
+
+- name: Check if the File command is present
+  register: subcommand_list
+  shell:
+    cmd: >-
+      {{ validation_command }} --help
+    executable: /bin/bash
+
+- name: Execute the file command tests
+  block:
+    - name: Copy files to run
+      template:
+        src: './templates/file-template.j2'
+        dest: rendered_file_{{ ansible_loop.index }}.yaml
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+      loop: "{{ test_arguments_run_from_file }}"
+      loop_control:
+        extended: true
+      register: rendered_files
+
+    - name: Run validations from the File
+      include_tasks: file.yaml
+      vars:
+        files: "{{ rendered_files.results | map(attribute='dest') | list }}"
+  when: "'Include and exclude validations by' in subcommand_list.stdout"
+
+- name: Fail if something went wrong
+  fail:
+    msg: "One or more file runs have failed, check the log results for more information."
+  when: files_test_failure | default(False) | bool and "'file' in subcommand_list.stdout"

roles/validations/tasks/run.yaml

@@ -14,7 +14,7 @@
       shell:
         cmd: >-
           {{ validation_command }} run --validation {{ name.key }}
-          {{ validation_dir }} {{ ansible_dir }}
+          {{ validation_dir_arg }} {{ ansible_dir_arg }}
           --inventory {{ inventory }}
           --output-log validation_{{ name.key }}_positive.log
           {{ execution_extra_args | default(name.value.extra_args) }}

roles/validations/tasks/run_extra_vars_file.yaml

@@ -8,7 +8,7 @@
   shell:
     cmd: >-
       {{ validation_command }} run --validation {{ name.key }}
-      {{ validation_dir }} {{ ansible_dir }}
+      {{ validation_dir_arg }} {{ ansible_dir_arg }}
       --inventory {{ inventory }}
       --output-log validation_{{ name.key }}_extra_vars_file.log
       --extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml

roles/validations/tasks/show_validation_info.yaml

@@ -2,7 +2,7 @@
 - name: Show Validation - correct id
   register: show_output
   shell:
-    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
+    cmd: "{{ validation_command }} show {{ validation_dir_arg }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
     executable: /bin/bash
 
 # Simulating a typo in the validation name
@@ -11,7 +11,7 @@
     - name: Show validations - incorrect id
       register: show_output_incorrect
       shell:
-        cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
+        cmd: "{{ validation_command }} show {{ validation_dir_arg }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
         executable: /bin/bash
       ignore_errors: true
     - fail:
@@ -22,7 +22,7 @@
   shell:
     cmd: >-
      {{ validation_command }} show parameter
-     --validation {{ name.key }} {{ validation_dir }}
+     --validation {{ name.key }} {{ validation_dir_arg }}
      --download {{ name.key }}_params.{{ format_type }}
      --format-output {{ format_type }}
     executable: /bin/bash

roles/validations/templates/file-template.j2

@@ -0,0 +1,34 @@
+---
+include_validation: {{ item.validation }}
+include_group: {{ item.validation_group }}
+include_category: {{ item.validation_category }}
+include_product: {{ item.validation_product }}
+exclude_validation: {{ item.exclude_validation }}
+exclude_group: {{ item.exclude_validation_group }}
+exclude_category: {{ item.exclude_validation_category }}
+exclude_product: {{ item.exclude_validation_product }}
+config: {{ validation_config | default('') }}
+inventory: {{ inventory }}
+validation-log-dir: {{ vf_log_dir }}
+output-log: {{ vf_log_dir }}/run-from-file.log
+# Checks if the following variables are defined, if the variables
+# don't have the value, based on the vars/main.yaml file, they are skipped.
+{% if item.extra_env_vars | default('') %}
+extra-env-vars: {{ item.extra_env_vars }}
+{% endif %}
+{% if item.extra_vars | default('') %}
+extra-vars: {{ item.extra_vars }}
+{% endif %}
+{% if item.limit_hosts | default('') %}
+limit: {{ item.limit_hosts.limit | default('') }}
+{% endif %}
+# Checks if the zuul virtualenv exists, if not default path is used instead.
+{% if is_virtualenv.stat.exists %}
+validation-dir: {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks
+ansible-base-dir: {{ zuul_work_virtualenv }}/share/ansible
+python-interpreter: {{ zuul_work_virtualenv }}/bin/python3
+{% else %}
+validation-dir: /usr/share/ansible/validation-playbooks
+ansible-base-dir: /usr/share/ansible
+python-interpreter: /usr/bin/python3
+{% endif %}

roles/validations/vars/main.yaml

@@ -186,3 +186,82 @@ validation_catalogue:
 #    - tls-everywhere-prep
 #    - undercloud-debug
 #    - undercloud-service-status
+#
+# List of dictionaries for testing 4 different versions of file-to-run the CLI file command
+# Each dictionary consists of different options for inclusion and exclusion
+# validations/groups/categories/products
+test_arguments_run_from_file:
+  # 1st valid file
+  # expected rc is 0
+  - validation:
+      - check-cpu
+    validation_group: []
+    validation_category: []
+    validation_product: []
+    exclude_validation:
+    exclude_validation_group: []
+    exclude_validation_category: []
+    exclude_validation_product:
+      - tripleo
+    validation-dir:
+    extra_vars:
+      minimal_cpu_count: 2
+  # 2nd valid file with 1 non-existent validation
+  # networking group should be run, expected rc is 1 due to the failing
+  # validations
+  - validation:
+      - check-cpu
+      - i-dont-exist
+    validation_group: []
+    validation_category:
+      - compute
+      - networking
+    validation_product: []
+    exclude_validation:
+      - fips-enabled
+    exclude_validation_group:
+      - prep
+    exclude_validation_category:
+      - compute
+    exclude_validation_product:
+      - rabbitmq
+      - tripleo
+    limit_hosts:
+      limit: [undercloud-0]
+  #  3rd valid file testing the Invalid operation: no validation to run
+  #  expected rc is 1
+  - validation:
+      - i-dont-exist
+    validation_group: []
+    validation_category: []
+    validation_product: []
+    exclude_validation:
+      - fips-enabled
+    exclude_validation_group:
+      - prep
+    exclude_validation_category:
+      - compute
+    exclude_validation_product:
+      - rabbitmq
+      - tripleo
+  #  4th valid file, testing the proper inclusion and exclusion
+  #  only networking group should run (except the dns and the ntp validations)
+  # expected rc is 1 due to the failed check-cpu validation
+  - validation:
+      - check-cpu
+    validation_group:
+    validation_category:
+      - networking
+    validation_product:
+      - tripleo
+    exclude_validation:
+      - fips-enabled
+      - dns
+      - ntp
+    exclude_validation_group:
+    exclude_validation_category:
+    exclude_validation_product:
+      - rabbitmq
+      - tripleo
+    extra_vars:
+      minimal_cpu_count: 8000