From 7b09b0ad5cccefdb8b9358047a94bf85a038fc3a Mon Sep 17 00:00:00 2001 From: mcgonago Date: Mon, 21 Aug 2023 16:31:30 -0400 Subject: [PATCH] [WIP] [update] verify hardcode values for update validations This is a WIP to help verify results against [1] when merged and available in next triplo-validations library. This patch will be abandoned once testing is complete. [1] https://review.opendev.org/c/openstack/tripleo-validations/+/892168 Change-Id: I305baa5533f2d64c84dca98f6b9183bfa40c7a03 --- .../validation_group_run_post_update.yaml | 85 +++++++++++++++++ ...validation_group_run_post_update_blue.yaml | 77 +++++++++++++++ .../validation_group_run_pre_update.yaml | 93 +++++++++++++++++++ .../validation_group_run_pre_update_blue.yaml | 85 +++++++++++++++++ tasks/update/main.yml | 8 +- 5 files changed, 344 insertions(+), 4 deletions(-) create mode 100644 tasks/common/validation_group_run_post_update.yaml create mode 100644 tasks/common/validation_group_run_post_update_blue.yaml create mode 100644 tasks/common/validation_group_run_pre_update.yaml create mode 100644 tasks/common/validation_group_run_pre_update_blue.yaml diff --git a/tasks/common/validation_group_run_post_update.yaml b/tasks/common/validation_group_run_post_update.yaml new file mode 100644 index 00000000..8a6ee09d --- /dev/null +++ b/tasks/common/validation_group_run_post_update.yaml @@ -0,0 +1,85 @@ +--- +# validation_group: validation group to execute +# skiplist_validations: comma separated string of the validations to be skipped in a group +# skiplist_validations_stage: comma separated string of the validations to be skipped in a group defined for a stage, added to the list. +# validation_allowed_groups: list of allowed groups to run +# validation_args: string containing extra arguments for the validation command. (defaults to empty string) +- block: + - name: "Retrieve validations for group {{ validation_group }}" + command: "validation list --group {{ validation_group }} -f value -c ID" + register: validations_in_group + - name: Set fact validations_filtered which removes validations from skiplist + set_fact: + validations_filtered: "{{ validations_in_group.stdout_lines | difference( skiplist_validations.split(',') + skiplist_validations_stage.split(',') ) }}" + - name: "Running validations {{ validations_filtered }}" + register: validations_result + shell: | + set -o pipefail + source {{ undercloud_rc }} + validation run --validation \ + verify-package,\ + undercloud-disabled-services,\ + openstack-endpoints,\ + undercloud-service-status,\ + rabbitmq-limits,\ + check-rsyslog,\ + ntp,\ + service-status,\ + validate-selinux,\ + stonith-exists,\ + overcloud-service-status,\ + ceph-health,\ + ovs-dpdk-pmd-cpus-check,\ + check-kernel-version,\ + tls-everywhere-post-deployment,\ + check-manila-policy-file,\ + tripleo-haproxy,\ + container-status,\ + controller-token,\ + controller-ulimits,\ + fips-enabled,\ + frr-status,\ + pacemaker-status,\ + healthcheck-service-status,\ + image-serve,\ + mysql-open-files-limit,\ + neutron-sanity-check,\ + nfv-ovsdpdk-zero-packet-loss-check,\ + nova-event-callback,\ + nova-svirt \ + --inventory {{ upgrade_validation_inventory }} \ + {{ validations_extra_args }} 2>&1 {{ timestamper_cmd }} >> validation-{{ validation_group }}.log + when: + - "validations_filtered|length > 0" + args: + executable: /usr/bin/bash + - name: Validation group not executed + debug: + msg: "Validation group {{ validation_group }} is skipped." + when: + - "validations_filtered|length == 0" + rescue: + - name: Retrieve the validations results + command: validation history list -f json -c UUID -c Status + register: validation_history + - name: Set fact validation_status with the validations results + set_fact: + validations_failed: "{{ validation_history.stdout|default('')|from_json|json_query(\"[?Status != 'PASSED'].UUID\") }}" + when: validation_history is defined + - name: Log all the validations failed + shell: | + validation show {{ item }} &>> validation-{{ validation_group }}-failed.log + loop: "{{ validations_failed }}" + when: + - "validations_failed|length > 0" + - name: Fail if some validation is not PASSED + fail: + msg: "Validation failed: check the log in validation-{{ validation_group }}-failed.log." + when: + - "validations_failed|length > 0" + - name: Fail if the validations command didn't succeed + fail: + msg: "Validations failed: check the log in validation-{{ validation_group }}.log." + when: + - validations_result is not succeeded + when: "validation_group in validation_allowed_groups" diff --git a/tasks/common/validation_group_run_post_update_blue.yaml b/tasks/common/validation_group_run_post_update_blue.yaml new file mode 100644 index 00000000..8ff69868 --- /dev/null +++ b/tasks/common/validation_group_run_post_update_blue.yaml @@ -0,0 +1,77 @@ +--- +# validation_group: validation group to execute +# skiplist_validations: comma separated string of the validations to be skipped in a group +# skiplist_validations_stage: comma separated string of the validations to be skipped in a group defined for a stage, added to the list. +# validation_allowed_groups: list of allowed groups to run +# validation_args: string containing extra arguments for the validation command. (defaults to empty string) +- block: + - name: "Retrieve validations for group {{ validation_group }}" + command: "validation list --group {{ validation_group }} -f value -c ID" + register: validations_in_group + - name: Set fact validations_filtered which removes validations from skiplist + set_fact: + validations_filtered: "{{ validations_in_group.stdout_lines | difference( skiplist_validations.split(',') + skiplist_validations_stage.split(',') ) }}" + - name: "Running validations {{ validations_filtered }}" + register: validations_result + shell: | + set -o pipefail + source {{ undercloud_rc }} + validation run --validation \ + verify-package,\ + ceph-health,\ + undercloud-disabled-services,\ + openstack-endpoints,\ + undercloud-service-status,\ + rabbitmq-limits,\ + check-rsyslog,\ + ntp,\ + service-status,\ + validate-selinux,\ + stonith-exists,\ + tripleo-haproxy,\ + container-status,\ + controller-token,\ + controller-ulimits,\ + fips-enabled,\ + healthcheck-service-status,\ + image-serve,\ + mysql-open-files-limit,\ + neutron-sanity-check,\ + nova-event-callback,\ + nova-svirt \ + --inventory {{ upgrade_validation_inventory }} \ + {{ validations_extra_args }} 2>&1 {{ timestamper_cmd }} >> validation-{{ validation_group }}.log + when: + - "validations_filtered|length > 0" + args: + executable: /usr/bin/bash + - name: Validation group not executed + debug: + msg: "Validation group {{ validation_group }} is skipped." + when: + - "validations_filtered|length == 0" + rescue: + - name: Retrieve the validations results + command: validation history list -f json -c UUID -c Status + register: validation_history + - name: Set fact validation_status with the validations results + set_fact: + validations_failed: "{{ validation_history.stdout|default('')|from_json|json_query(\"[?Status != 'PASSED'].UUID\") }}" + when: validation_history is defined + - name: Log all the validations failed + shell: | + validation show {{ item }} &>> validation-{{ validation_group }}-failed.log + loop: "{{ validations_failed }}" + when: + - "validations_failed|length > 0" + - name: Fail if some validation is not PASSED + fail: + msg: "Validation failed: check the log in validation-{{ validation_group }}-failed.log." + when: + - "validations_failed|length > 0" + - name: Fail if the validations command didn't succeed + fail: + msg: "Validations failed: check the log in validation-{{ validation_group }}.log." + when: + - validations_result is not succeeded + when: "validation_group in validation_allowed_groups" diff --git a/tasks/common/validation_group_run_pre_update.yaml b/tasks/common/validation_group_run_pre_update.yaml new file mode 100644 index 00000000..ade4db74 --- /dev/null +++ b/tasks/common/validation_group_run_pre_update.yaml @@ -0,0 +1,93 @@ +--- +# validation_group: validation group to execute +# skiplist_validations: comma separated string of the validations to be skipped in a group +# skiplist_validations_stage: comma separated string of the validations to be skipped in a group defined for a stage, added to the list. +# validation_allowed_groups: list of allowed groups to run +# validation_args: string containing extra arguments for the validation command. (defaults to empty string) +- block: + - name: "Retrieve validations for group {{ validation_group }}" + command: "validation list --group {{ validation_group }} -f value -c ID" + register: validations_in_group + - name: Set fact validations_filtered which removes validations from skiplist + set_fact: + validations_filtered: "{{ validations_in_group.stdout_lines | difference( skiplist_validations.split(',') + skiplist_validations_stage.split(',') ) }}" + - name: "Running validations {{ validations_filtered }}" + register: validations_result + shell: | + set -o pipefail + source {{ undercloud_rc }} + validation run --validation \ + verify-package,\ + undercloud-sysctl,\ + package-version,\ + undercloud-disabled-services,\ + openstack-endpoints,\ + undercloud-service-status,\ + rabbitmq-limits,\ + check-rsyslog,\ + ntp,\ + service-status,\ + validate-selinux,\ + stonith-exists,\ + overcloud-service-status,\ + ceph-health,\ + ovs-dpdk-pmd-cpus-check,\ + check-kernel-version,\ + tls-everywhere-post-deployment,\ + check-manila-policy-file,\ + tripleo-haproxy,\ + container-status,\ + controller-token,\ + controller-ulimits,\ + fips-enabled,\ + frr-status,\ + pacemaker-status,\ + healthcheck-service-status,\ + image-serve,\ + mysql-open-files-limit,\ + neutron-sanity-check,\ + nfv-ovsdpdk-zero-packet-loss-check,\ + nova-event-callback,\ + nova-svirt \ + repos,\ + check-for-dangling-images,\ + check-reboot,\ + system-encoding,\ + compute-tsx,\ + undercloud-ipa-server-check \ + --inventory {{ upgrade_validation_inventory }} \ + {{ validations_extra_args }} 2>&1 {{ timestamper_cmd }} >> validation-{{ validation_group }}.log + when: + - "validations_filtered|length > 0" + args: + executable: /usr/bin/bash + - name: Validation group not executed + debug: + msg: "Validation group {{ validation_group }} is skipped." + when: + - "validations_filtered|length == 0" + rescue: + - name: Retrieve the validations results + command: validation history list -f json -c UUID -c Status + register: validation_history + - name: Set fact validation_status with the validations results + set_fact: + validations_failed: "{{ validation_history.stdout|default('')|from_json|json_query(\"[?Status != 'PASSED'].UUID\") }}" + when: validation_history is defined + - name: Log all the validations failed + shell: | + validation show {{ item }} &>> validation-{{ validation_group }}-failed.log + loop: "{{ validations_failed }}" + when: + - "validations_failed|length > 0" + - name: Fail if some validation is not PASSED + fail: + msg: "Validation failed: check the log in validation-{{ validation_group }}-failed.log." + when: + - "validations_failed|length > 0" + - name: Fail if the validations command didn't succeed + fail: + msg: "Validations failed: check the log in validation-{{ validation_group }}.log." + when: + - validations_result is not succeeded + when: "validation_group in validation_allowed_groups" diff --git a/tasks/common/validation_group_run_pre_update_blue.yaml b/tasks/common/validation_group_run_pre_update_blue.yaml new file mode 100644 index 00000000..f93a8052 --- /dev/null +++ b/tasks/common/validation_group_run_pre_update_blue.yaml @@ -0,0 +1,85 @@ +--- +# validation_group: validation group to execute +# skiplist_validations: comma separated string of the validations to be skipped in a group +# skiplist_validations_stage: comma separated string of the validations to be skipped in a group defined for a stage, added to the list. +# validation_allowed_groups: list of allowed groups to run +# validation_args: string containing extra arguments for the validation command. (defaults to empty string) +- block: + - name: "Retrieve validations for group {{ validation_group }}" + command: "validation list --group {{ validation_group }} -f value -c ID" + register: validations_in_group + - name: Set fact validations_filtered which removes validations from skiplist + set_fact: + validations_filtered: "{{ validations_in_group.stdout_lines | difference( skiplist_validations.split(',') + skiplist_validations_stage.split(',') ) }}" + - name: "Running validations {{ validations_filtered }}" + register: validations_result + shell: | + set -o pipefail + source {{ undercloud_rc }} + validation run --validation \ + verify-package,\ + ceph-health,\ + undercloud-disabled-services,\ + openstack-endpoints,\ + undercloud-service-status,\ + rabbitmq-limits,\ + check-rsyslog,\ + ntp,\ + service-status,\ + validate-selinux,\ + stonith-exists,\ + tripleo-haproxy,\ + container-status,\ + controller-token,\ + controller-ulimits,\ + fips-enabled,\ + healthcheck-service-status,\ + image-serve,\ + mysql-open-files-limit,\ + neutron-sanity-check,\ + nova-event-callback,\ + nova-svirt, \ + undercloud-sysctl,\ + package-version,\ + repos,\ + check-for-dangling-images,\ + check-reboot,\ + system-encoding,\ + compute-tsx,\ + undercloud-ipa-server-check \ + --inventory {{ upgrade_validation_inventory }} \ + {{ validations_extra_args }} 2>&1 {{ timestamper_cmd }} >> validation-{{ validation_group }}.log + when: + - "validations_filtered|length > 0" + args: + executable: /usr/bin/bash + - name: Validation group not executed + debug: + msg: "Validation group {{ validation_group }} is skipped." + when: + - "validations_filtered|length == 0" + rescue: + - name: Retrieve the validations results + command: validation history list -f json -c UUID -c Status + register: validation_history + - name: Set fact validation_status with the validations results + set_fact: + validations_failed: "{{ validation_history.stdout|default('')|from_json|json_query(\"[?Status != 'PASSED'].UUID\") }}" + when: validation_history is defined + - name: Log all the validations failed + shell: | + validation show {{ item }} &>> validation-{{ validation_group }}-failed.log + loop: "{{ validations_failed }}" + when: + - "validations_failed|length > 0" + - name: Fail if some validation is not PASSED + fail: + msg: "Validation failed: check the log in validation-{{ validation_group }}-failed.log." + when: + - "validations_failed|length > 0" + - name: Fail if the validations command didn't succeed + fail: + msg: "Validations failed: check the log in validation-{{ validation_group }}.log." + when: + - validations_result is not succeeded + when: "validation_group in validation_allowed_groups" diff --git a/tasks/update/main.yml b/tasks/update/main.yml index b8c3e68f..c6d48509 100644 --- a/tasks/update/main.yml +++ b/tasks/update/main.yml @@ -27,7 +27,7 @@ import_tasks: ../common/undercloud_prerequisites.yaml - name: run pre-update validation - import_tasks: ../common/validation_group_run.yaml + import_tasks: ../common/validation_group_run_pre_update_blue.yaml vars: validation_group: "pre-update" validation_allowed_groups: "{{ updates_validations_groups }}" @@ -95,7 +95,7 @@ - overcloud_update_prepare - name: run pre-update-prepare validation - import_tasks: ../common/validation_group_run.yaml + import_tasks: ../common/validation_group_run_pre_update_blue.yaml vars: validation_group: "pre-update-prepare" validation_allowed_groups: "{{ updates_validations_groups }}" @@ -186,7 +186,7 @@ - overcloud_update_run - name: run pre-update-run validation - import_tasks: ../common/validation_group_run.yaml + import_tasks: ../common/validation_group_run_pre_update_blue.yaml vars: validation_group: "pre-update-run" validation_allowed_groups: "{{ updates_validations_groups }}" @@ -269,7 +269,7 @@ - log_stages|bool - name: run post-update validation - import_tasks: ../common/validation_group_run.yaml + import_tasks: ../common/validation_group_run_post_update_blue.yaml vars: validation_group: "post-update" validation_allowed_groups: "{{ updates_validations_groups }}"