Validation Framework functional test expansion

Issues present in the existing tests of the validation framework
runtime and CLI behavior were resolved. New tests were added to ensure
correct function of all of the most important commands, subcommands
and combinations of parameters.

* conflicting 'run_validations' variable was removed
* show command tests were enabled
* all test blocks are descriptively named
* tests for listing installed validations in various formats
* tests for running validations with parameters from a file
* tests for history display, both with and without limit
* tests for detailed validation information display
* tests for validation parameter display and download

Validation log fetch was modified to retrieve more data.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: Ic206e39f792b31d7f2adff89e4d38a2e8a656733
This commit is contained in:
Jiri Podivin 2021-08-03 09:38:28 +02:00
parent f7bbf218ee
commit 7c2026616b
9 changed files with 232 additions and 12 deletions

View File

@ -35,7 +35,7 @@
- name: Find validations data
find:
paths: "{{ output_dir }}"
patterns: "*.json,*.log"
patterns: "*.json,*.log,*.yaml"
register: validation_json
- name: Collect Validation logs

View File

@ -1,6 +1,36 @@
---
- name: List Validation
register: list_output
- name: List Validations - all - to file
shell:
cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1"
executable: /bin/bash
when: val_format == "json"
- name: List Validations - all - to stdout - {{ val_format }}
shell:
cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}"
executable: /bin/bash
# Metadata dependent list output
- name: List Validations - group - to stdout - {{ val_format }}
shell:
cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}"
executable: /bin/bash
loop: "{{ validation_metadata.group }}"
loop_control:
loop_var: val_group
- name: " List Validations - category - to stdout - {{ val_format }} "
shell:
cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}"
executable: /bin/bash
loop: "{{ validation_metadata.category }}"
loop_control:
loop_var: val_category
- name: "List Validations - product - to stdout - {{ val_format }}"
shell:
cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}"
executable: /bin/bash
loop: "{{ validation_metadata.product }}"
loop_control:
loop_var: val_product

View File

@ -0,0 +1,52 @@
---
# The subcommand used is 'show history' but it is implemented
# as a subclass of Lister and it formats the results as such.
# Both tests use regex to keep only lines starting with UUID[1].
# As every validation run has UUID assigned and the 'value' output format
# places it's octal form in the first column, it is possible to use it to
# match only records about validation runs, and remove the rest.
# [1]https://datatracker.ietf.org/doc/html/rfc4122#section-4.1
- name: List all history
register: list_all_history_output
shell:
cmd: >-
{{ validation_command }} {{ history_command }} -f value 2>&1
| grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
| tee {{ val_working_dir }}/full_validation_history.log
executable: /bin/bash
- name: List truncated history
register: list_truncated_history_output
shell:
cmd: >-
{{ validation_command }} {{ history_command }} --limit 1 -f value 2>&1
| grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
| tee {{ val_working_dir }}/truncated_validation_history.log
executable: /bin/bash
- name: Verify history output
block:
# To ensure that we are getting the right number of validation runs
# we are querying the relevant item of the 'validations_list', for the number of 'extra_args' entries.
# As all validations defined in the 'validations_list' have 'extra_args' defined for both normal,
# and false positive run, we can use the number of 'extra_args' keys as an indication
# of the validations runs that were supposed to occur.
# Please note that this assertion will not hold, if the format of the default_vars changes substantially.
- name: Verify full history output
fail:
msg: >
The history output length {{ list_all_history_output.stdout_lines | length }}
doesn't match the number of expected validations runs {{ expected_history_length }}.
when: (list_all_history_output.stdout_lines | length) != (expected_history_length | int)
vars:
expected_history_length: "{{ validations_list[validation_component] | string | regex_findall('extra_args') | length }}"
- name: Verify truncated history output
fail:
msg: >
The number of history items displayed is {{ list_truncated_history_output.stdout_lines | length }}
but it should be 1.
when: (list_truncated_history_output.stdout_lines | length) != 1
when:
- run_validation|default(false)|bool
- validation_component | length > 0

View File

@ -43,7 +43,8 @@
validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
when: not is_virtualenv.stat.exists
- include_tasks: run.yaml
- name: Run validations
include_tasks: run.yaml
vars:
name: "{{ item }}"
when:
@ -51,7 +52,16 @@
- validation_component | length > 0
with_dict: "{{ validations_list[validation_component] }}"
- include_tasks: show.yaml
- name: List validations
include_tasks: list.yaml
vars:
val_format: "{{ tested_format }}"
loop: "{{ validation_list_formats }}"
loop_control:
loop_var: tested_format
- name: Show validation run results
include_tasks: show_results.yaml
vars:
name: "{{ item }}"
when:
@ -59,4 +69,26 @@
- validation_component | length > 0
with_dict: "{{ validations_list[validation_component] }}"
- include_tasks: list.yaml
- name: Show validation
include_tasks: show_validation_info.yaml
vars:
name: "{{ item }}"
when:
- run_validation|default(false)|bool
- validation_component | length > 0
with_dict: "{{ validations_list[validation_component] }}"
- name: List history
include_tasks: list_validation_history.yaml
vars:
history_command: "{{'show history' if validation_command == 'openstack tripleo validator' else 'history list'}}"
- name: Run validations with extra vars file
include_tasks: run_extra_vars_file.yaml
vars:
name: "{{ item }}"
extra_vars_uuid: "{{ 'extra vars for tests' | to_uuid }}"
when:
- run_validation|default(false)|bool
- validation_component | length > 0
with_dict: "{{ validations_list[validation_component] }}"

View File

@ -0,0 +1,41 @@
---
- name: Create extra vars file
shell:
cmd: "echo -e 'minimal_cpu_count: 2\nminimal_ram_gb: 2\n' > {{ extra_vars_uuid }}extra_vars.yaml"
executable: /bin/bash
- name: Run validations with extra vars file
shell:
cmd: >-
{{ validation_command }} run --validation {{ name.key }}
{{ validation_dir }} {{ ansible_dir }}
--inventory {{ inventory }}
--output-log validation_{{ name.key }}_extra_vars_file.log
--extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml
{{ name.value.extra_env_args }}
executable: /bin/bash
- name: Get Run results
block:
- name: Get run results
register: result
shell:
cmd: "cat validation_{{ name.key }}_extra_vars_file.log"
executable: /bin/bash
- name: Get json data
set_fact:
jsondata: "{{ result.stdout | from_json }}"
- name: Get Validations Status
set_fact:
status: "{{ jsondata | json_query(jsonres) }}"
vars:
jsonres: 'results[*].Status'
- fail:
msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}"
when: validation_status != "PASSED"
loop: "{{ status }}"
loop_control:
loop_var: validation_status

View File

@ -1,6 +0,0 @@
---
- name: Show Validation
register: show_output
shell:
cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
executable: /bin/bash

View File

@ -0,0 +1,21 @@
---
- name: Get run UUID
block:
- name: Get run results file
register: result
shell:
cmd: "cat validation_{{ name.key }}_positive.log"
executable: /bin/bash
- name: Get uuid from log
set_fact:
validation_run_uuids: "{{ result.stdout | from_json | json_query(uuid_selector) }}"
vars:
uuid_selector: 'results[*].UUID'
- name: Show Validation run results
shell:
cmd: "{{ validation_command }} history get {{ run_uuid }} --full > {{ val_working_dir }}/show_run.log 2>&1"
executable: /bin/bash
loop: "{{ validation_run_uuids }}"
loop_control:
loop_var: run_uuid

View File

@ -0,0 +1,33 @@
---
- name: Show Validation - correct id
register: show_output
shell:
cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
executable: /bin/bash
# Simulating a typo in the validation name
- name: Show Validation - incorrect id
block:
- name: Show validations - incorrect id
register: show_output_incorrect
shell:
cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
executable: /bin/bash
ignore_errors: true
- fail:
msg: "The expected error message was not displayed."
when: "'Validation chuck-cpu not found' not in show_output_incorrect.stdout"
- name: Show Validation parameters
shell:
cmd: >-
{{ validation_command }} show parameter
--validation {{ name.key }} {{ validation_dir }}
--download {{ name.key }}_params.{{ format_type }}
--format-output {{ format_type }}
executable: /bin/bash
loop:
- json
- yaml
loop_control:
loop_var: format_type

View File

@ -39,3 +39,20 @@ validations_group:
- compute
network:
- network
validation_metadata:
group:
- no-op
- prep
category:
- storage
- container
product:
- tripleo
validation_list_formats:
- csv
- json
- table
- value
- yaml