diff --git a/roles/validations/defaults/main.yaml b/roles/validations/defaults/main.yaml index 6f9afe3..f8adff1 100644 --- a/roles/validations/defaults/main.yaml +++ b/roles/validations/defaults/main.yaml @@ -23,3 +23,5 @@ cli_command: "validation" run_validation: true execute_full_vf_catalogue: "{{ job.execute_full_vf_catalogue|default(false)|bool }}" vf_catalogue_overrides: "{{ ansible_user_dir }}/catalog_vars_override.yaml" +files_to_run_dest: "{{ ansible_user_dir }}" +files_test_failure: false diff --git a/roles/validations/tasks/execute_full_catalogue.yaml b/roles/validations/tasks/execute_full_catalogue.yaml index c24ff5b..44242af 100644 --- a/roles/validations/tasks/execute_full_catalogue.yaml +++ b/roles/validations/tasks/execute_full_catalogue.yaml @@ -7,7 +7,7 @@ shell: cmd: >- {{ validation_command }} run --validation {{ validation_catalogue.full_list | join(",") }} - {{ validation_dir }} {{ ansible_dir }} + {{ validation_dir_arg }} {{ ansible_dir_arg }} --inventory {{ inventory }} --output-log validation_catalogue_execution.log {{ validation_catalogue.extra_args }} diff --git a/roles/validations/tasks/file.yaml b/roles/validations/tasks/file.yaml new file mode 100644 index 0000000..3390e71 --- /dev/null +++ b/roles/validations/tasks/file.yaml @@ -0,0 +1,118 @@ +--- +- name: Run validations from the YAML file + vars: + file_wrong_path: foo/bar.yaml + junitxml_path: /home/stack/logs + junitxml_wrong_path: /foo/bar + wrong_path_string: must be properly formatted + wrong_directory_string: No such file or directory + junitxml_missing_string: junitxml output disabled + validation_failed_string: have failed + validation_not_found_string: Following validations were not found + no_validation_run_string: No validation has been run + expected_result: 1 + block: + - debug: + msg: "{{ item }}" + loop: "{{ files }}" + + - name: Execute the file command + block: + - name: Passed validation test + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[0] }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: "{{ run_results.rc }} == 1" + + - name: Failed validation test + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[3] }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: run_results.rc != expected_result + + - name: Run with no executed validation + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[1] }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: + - run_results.rc != expected_result + + - name: Run with non-existent validation + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[2] }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: + - validation_not_found_string not in run_results.stdout + - validation_not_found_string not in run_results.stderr + + - name: Execute the file command with wrong path + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ file_wrong_path }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: + - wrong_path_string not in run_results.stdout + - wrong_path_string not in run_results.stderr + + - name: Execute the file command with junitxml output + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_path }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: + - wrong_directory_string in run_results.stdout + - junitxml_missing_string not in run_results.stdout + - wrong_directory_string in run_results.stderr + - junitxml_missing_string not in run_results.stderr + - "{{ run_results.rc }} == 1" + + - name: Execute the file command with wrong Junitxml path + ignore_errors: true + register: run_results + shell: + cmd: >- + {{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_wrong_path }} + executable: /bin/bash + + - set_fact: + files_test_failure: true + when: + - wrong_directory_string in run_results.stdout + - junitxml_missing_string not in run_results.stdout + - wrong_directory_string in run_results.stderr + - junitxml_missing_string not in run_results.stderr + - "{{ run_results.rc }} == 1" diff --git a/roles/validations/tasks/list.yaml b/roles/validations/tasks/list.yaml index fe6b236..6b95669 100644 --- a/roles/validations/tasks/list.yaml +++ b/roles/validations/tasks/list.yaml @@ -1,19 +1,19 @@ --- - name: List Validations - all - to file shell: - cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1" + cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f json > {{ val_working_dir }}/list.log 2>&1" executable: /bin/bash when: val_format == "json" - name: List Validations - all - to stdout - {{ val_format }} shell: - cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}" + cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f {{ val_format }}" executable: /bin/bash # Metadata dependent list output - name: List Validations - group - to stdout - {{ val_format }} shell: - cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}" + cmd: "{{ validation_command }} list {{ validation_dir_arg }} --group {{ val_group }} -f {{ val_format }}" executable: /bin/bash loop: "{{ validation_metadata.group }}" loop_control: @@ -21,7 +21,7 @@ - name: " List Validations - category - to stdout - {{ val_format }} " shell: - cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}" + cmd: "{{ validation_command }} list {{ validation_dir_arg }} --category {{ val_category }} -f {{ val_format }}" executable: /bin/bash loop: "{{ validation_metadata.category }}" loop_control: @@ -29,7 +29,7 @@ - name: "List Validations - product - to stdout - {{ val_format }}" shell: - cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}" + cmd: "{{ validation_command }} list {{ validation_dir_arg }} --product {{ val_product }} -f {{ val_format }}" executable: /bin/bash loop: "{{ validation_metadata.product }}" loop_control: diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml index 300383f..1155541 100644 --- a/roles/validations/tasks/main.yaml +++ b/roles/validations/tasks/main.yaml @@ -39,12 +39,22 @@ - name: Set Validation directory if virtualenv exists set_fact: - validation_dir: "--validation-dir {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks" + validation_dir: "{{ zuul_work_virtualenv }}/share/ansible/validation-playbooks" when: is_virtualenv.stat.exists -- name: Set Ansible base directory if virtualenv exists +- name: Set Validation directory argument if virtualenv exists set_fact: - ansible_dir: "--ansible-base-dir {{ zuul_work_virtualenv }}/share/ansible/" + validation_dir_arg: "--validation-dir {{ validation_dir }}" + when: is_virtualenv.stat.exists + +- name: Set Ansible base directory path if virtualenv exists + set_fact: + ansible_dir: "{{ zuul_work_virtualenv }}/share/ansible/" + when: is_virtualenv.stat.exists + +- name: Set Ansible base directory argument if virtualenv exists + set_fact: + ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}" when: is_virtualenv.stat.exists - name: Set commmand without virtualenv @@ -56,7 +66,22 @@ - name: Set validation dir without virtualenv set_fact: - validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks" + validation_dir: "/usr/share/ansible/validation-playbooks" + when: not is_virtualenv.stat.exists + +- name: Set validation dir argument without virtualenv + set_fact: + validation_dir_arg: "--validation-dir {{ validation_dir }}" + when: not is_virtualenv.stat.exists + +- name: Set Ansible base directory path withnout virtualenv + set_fact: + ansible_dir: "/usr/share/ansible/" + when: not is_virtualenv.stat.exists + +- name: Set Ansible base directory argument withnout virtualenv exists + set_fact: + ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}" when: not is_virtualenv.stat.exists - name: Set a valid inventory @@ -167,3 +192,39 @@ - name: Execute entire validations catalogue include_tasks: execute_full_catalogue.yaml when: execute_full_vf_catalogue + +- debug: + msg: "{{ item }}" + loop: "{{ test_arguments_run_from_file }}" + +- name: Check if the File command is present + register: subcommand_list + shell: + cmd: >- + {{ validation_command }} --help + executable: /bin/bash + +- name: Execute the file command tests + block: + - name: Copy files to run + template: + src: './templates/file-template.j2' + dest: rendered_file_{{ ansible_loop.index }}.yaml + mode: "0644" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + loop: "{{ test_arguments_run_from_file }}" + loop_control: + extended: true + register: rendered_files + + - name: Run validations from the File + include_tasks: file.yaml + vars: + files: "{{ rendered_files.results | map(attribute='dest') | list }}" + when: "'Include and exclude validations by' in subcommand_list.stdout" + +- name: Fail if something went wrong + fail: + msg: "One or more file runs have failed, check the log results for more information." + when: files_test_failure | default(False) | bool and "'file' in subcommand_list.stdout" diff --git a/roles/validations/tasks/run.yaml b/roles/validations/tasks/run.yaml index bfcdc8b..7af0776 100644 --- a/roles/validations/tasks/run.yaml +++ b/roles/validations/tasks/run.yaml @@ -14,7 +14,7 @@ shell: cmd: >- {{ validation_command }} run --validation {{ name.key }} - {{ validation_dir }} {{ ansible_dir }} + {{ validation_dir_arg }} {{ ansible_dir_arg }} --inventory {{ inventory }} --output-log validation_{{ name.key }}_positive.log {{ execution_extra_args | default(name.value.extra_args) }} diff --git a/roles/validations/tasks/run_extra_vars_file.yaml b/roles/validations/tasks/run_extra_vars_file.yaml index 41291fb..261eb77 100644 --- a/roles/validations/tasks/run_extra_vars_file.yaml +++ b/roles/validations/tasks/run_extra_vars_file.yaml @@ -8,7 +8,7 @@ shell: cmd: >- {{ validation_command }} run --validation {{ name.key }} - {{ validation_dir }} {{ ansible_dir }} + {{ validation_dir_arg }} {{ ansible_dir_arg }} --inventory {{ inventory }} --output-log validation_{{ name.key }}_extra_vars_file.log --extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml diff --git a/roles/validations/tasks/show_validation_info.yaml b/roles/validations/tasks/show_validation_info.yaml index 09bfa49..566d55b 100644 --- a/roles/validations/tasks/show_validation_info.yaml +++ b/roles/validations/tasks/show_validation_info.yaml @@ -2,7 +2,7 @@ - name: Show Validation - correct id register: show_output shell: - cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1" + cmd: "{{ validation_command }} show {{ validation_dir_arg }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1" executable: /bin/bash # Simulating a typo in the validation name @@ -11,7 +11,7 @@ - name: Show validations - incorrect id register: show_output_incorrect shell: - cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log" + cmd: "{{ validation_command }} show {{ validation_dir_arg }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log" executable: /bin/bash ignore_errors: true - fail: @@ -22,7 +22,7 @@ shell: cmd: >- {{ validation_command }} show parameter - --validation {{ name.key }} {{ validation_dir }} + --validation {{ name.key }} {{ validation_dir_arg }} --download {{ name.key }}_params.{{ format_type }} --format-output {{ format_type }} executable: /bin/bash diff --git a/roles/validations/templates/file-template.j2 b/roles/validations/templates/file-template.j2 new file mode 100644 index 0000000..aa966ec --- /dev/null +++ b/roles/validations/templates/file-template.j2 @@ -0,0 +1,34 @@ +--- +include_validation: {{ item.validation }} +include_group: {{ item.validation_group }} +include_category: {{ item.validation_category }} +include_product: {{ item.validation_product }} +exclude_validation: {{ item.exclude_validation }} +exclude_group: {{ item.exclude_validation_group }} +exclude_category: {{ item.exclude_validation_category }} +exclude_product: {{ item.exclude_validation_product }} +config: {{ validation_config | default('') }} +inventory: {{ inventory }} +validation-log-dir: {{ vf_log_dir }} +output-log: {{ vf_log_dir }}/run-from-file.log +# Checks if the following variables are defined, if the variables +# don't have the value, based on the vars/main.yaml file, they are skipped. +{% if item.extra_env_vars | default('') %} +extra-env-vars: {{ item.extra_env_vars }} +{% endif %} +{% if item.extra_vars | default('') %} +extra-vars: {{ item.extra_vars }} +{% endif %} +{% if item.limit_hosts | default('') %} +limit: {{ item.limit_hosts.limit | default('') }} +{% endif %} +# Checks if the zuul virtualenv exists, if not default path is used instead. +{% if is_virtualenv.stat.exists %} +validation-dir: {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks +ansible-base-dir: {{ zuul_work_virtualenv }}/share/ansible +python-interpreter: {{ zuul_work_virtualenv }}/bin/python3 +{% else %} +validation-dir: /usr/share/ansible/validation-playbooks +ansible-base-dir: /usr/share/ansible +python-interpreter: /usr/bin/python3 +{% endif %} diff --git a/roles/validations/vars/main.yaml b/roles/validations/vars/main.yaml index 2cf3f6e..fce3ebc 100644 --- a/roles/validations/vars/main.yaml +++ b/roles/validations/vars/main.yaml @@ -186,3 +186,82 @@ validation_catalogue: # - tls-everywhere-prep # - undercloud-debug # - undercloud-service-status +# +# List of dictionaries for testing 4 different versions of file-to-run the CLI file command +# Each dictionary consists of different options for inclusion and exclusion +# validations/groups/categories/products +test_arguments_run_from_file: + # 1st valid file + # expected rc is 0 + - validation: + - check-cpu + validation_group: [] + validation_category: [] + validation_product: [] + exclude_validation: + exclude_validation_group: [] + exclude_validation_category: [] + exclude_validation_product: + - tripleo + validation-dir: + extra_vars: + minimal_cpu_count: 2 + # 2nd valid file with 1 non-existent validation + # networking group should be run, expected rc is 1 due to the failing + # validations + - validation: + - check-cpu + - i-dont-exist + validation_group: [] + validation_category: + - compute + - networking + validation_product: [] + exclude_validation: + - fips-enabled + exclude_validation_group: + - prep + exclude_validation_category: + - compute + exclude_validation_product: + - rabbitmq + - tripleo + limit_hosts: + limit: [undercloud-0] + # 3rd valid file testing the Invalid operation: no validation to run + # expected rc is 1 + - validation: + - i-dont-exist + validation_group: [] + validation_category: [] + validation_product: [] + exclude_validation: + - fips-enabled + exclude_validation_group: + - prep + exclude_validation_category: + - compute + exclude_validation_product: + - rabbitmq + - tripleo + # 4th valid file, testing the proper inclusion and exclusion + # only networking group should run (except the dns and the ntp validations) + # expected rc is 1 due to the failed check-cpu validation + - validation: + - check-cpu + validation_group: + validation_category: + - networking + validation_product: + - tripleo + exclude_validation: + - fips-enabled + - dns + - ntp + exclude_validation_group: + exclude_validation_category: + exclude_validation_product: + - rabbitmq + - tripleo + extra_vars: + minimal_cpu_count: 8000