From 1e9a27a2c353c9e2b5900a62674142549040b970 Mon Sep 17 00:00:00 2001 From: Raoul Scarazzini Date: Wed, 8 Feb 2017 13:25:55 -0500 Subject: [PATCH] Collect and manage logs when validating HA This commit manages all the stdout and stderr status of each HA validation test. In this way it will be possible to have all the logs related to the tests in one place (on the undercloud) that will be collected by the collect_logs role and so accessible from the external location where the logs will be stored. Change-Id: Ie803a920a17bd2f24ccd89f733781dd7e4d07fdd --- playbooks/overcloud-validate-ha.yml | 2 +- roles/validate-ha/defaults/main.yml | 3 +- roles/validate-ha/tasks/main.yml | 124 +++++++++++++++++++++------- 3 files changed, 96 insertions(+), 33 deletions(-) diff --git a/playbooks/overcloud-validate-ha.yml b/playbooks/overcloud-validate-ha.yml index 026668ae4..f097428a2 100644 --- a/playbooks/overcloud-validate-ha.yml +++ b/playbooks/overcloud-validate-ha.yml @@ -1,7 +1,7 @@ --- - name: Validate overcloud HA status - hosts: localhost + hosts: undercloud gather_facts: no roles: - validate-ha diff --git a/roles/validate-ha/defaults/main.yml b/roles/validate-ha/defaults/main.yml index 011fd36e3..b02941b29 100644 --- a/roles/validate-ha/defaults/main.yml +++ b/roles/validate-ha/defaults/main.yml @@ -1,6 +1,7 @@ --- -overcloud_working_dir: /home/heat-admin +overcloud_working_dir: "/home/heat-admin" +validate_ha_logs_dir: "{{ working_dir }}/validate_ha_logs" environment_file: environment.j2 apply_workarounds: false diff --git a/roles/validate-ha/tasks/main.yml b/roles/validate-ha/tasks/main.yml index 1fbac563d..5e6e9c754 100644 --- a/roles/validate-ha/tasks/main.yml +++ b/roles/validate-ha/tasks/main.yml @@ -1,13 +1,11 @@ --- - name: Creating the environment file on undercloud - delegate_to: undercloud template: src: "{{ environment_file }}" dest: "{{ working_dir }}/environment" mode: 0600 - name: Load the workarounds script on the undercloud - delegate_to: undercloud template: src: "{{ workarounds_script }}" dest: "{{ working_dir }}/workarounds.sh" @@ -15,7 +13,6 @@ when: apply_workarounds - name: Execute workarounds script on the undercloud - delegate_to: undercloud shell: > "{{ working_dir }}/workarounds.sh" when: apply_workarounds @@ -34,44 +31,109 @@ dir: "vars" files_matching: "test_list_{{ release }}.yml" -- name: HA test failed actions (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_check-failed-actions > {{ overcloud_working_dir }}/test_ha_failed-actions.log 2>&1 +- name: Create directory on the undercloud to store test results + file: path={{ validate_ha_logs_dir }} state=directory + +# Test: failed actions +- block: + - name: HA test - Failed actions (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_check-failed-actions + register: test_ha_failed_actions_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_failed_actions_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_failed_actions_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_failed_actions_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_failed_actions_stderr.log" when: test_ha_failed_actions|bool -- name: HA test Master/Slave core resource stop and start (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_master-slave -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_master-slave > {{ overcloud_working_dir }}/test_ha_master-slave.log 2>&1 +# Test: Master/Slave +- block: + - name: HA test - Master/Slave core resource stop and start (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_master-slave -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_master-slave + register: test_ha_master_slave_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_master_slave_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_master_slave_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_master_slave_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_master_slave_stderr.log" when: test_ha_master_slave|bool -- name: HA test Keystone removal (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_keystone-constraint-removal -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_keystone-constraint-removal > {{ overcloud_working_dir }}/test_ha_keystone-constraint-removal.log 2>&1 +# Test: Keystone removal +- block: + - name: HA test Keystone removal (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_keystone-constraint-removal -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_keystone-constraint-removal + register: test_ha_keystone_constraint_removal_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_keystone_constraint_removal_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_keystone_constraint_removal_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_keystone_constraint_removal_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_keystone_constraint_removal_stderr.log" when: test_ha_keystone_constraint_removal|bool -- name: HA test NG A (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-a -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light > {{ overcloud_working_dir }}/test_ha_ng-a.log 2>&1 +# Test: NG A +- block: + - name: HA test NG A (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-a -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light + register: test_ha_ng_a_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_ng_a_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_a_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_ng_a_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_a_stderr.log" when: test_ha_ng_a|bool -- name: HA test NG B (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-b -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light > {{ overcloud_working_dir }}/test_ha_ng-b.log 2>&1 +# Test: NG B +- block: + - name: HA test NG B (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-b -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light + register: test_ha_ng_b_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_ng_b_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_b_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_ng_b_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_b_stderr.log" when: test_ha_ng_b|bool -- name: HA test NG C (overcloud) - delegate_to: overcloud-controller-0 - shell: > - {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-c -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light > {{ overcloud_working_dir }}/test_ha_ng-c.log 2>&1 +# Test: NG C +- block: + - name: HA test NG C (overcloud) + delegate_to: overcloud-controller-0 + shell: > + {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/test/test_pacemaker-light-c -r {{ overcloud_working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_pacemaker-light + register: test_ha_ng_c_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_ng_c_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_c_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_ng_c_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_ng_c_stderr.log" when: test_ha_ng_c|bool -- name: HA Test instance deploy on the overcloud (undercloud) - delegate_to: undercloud - shell: > - {{ working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ working_dir }}/tripleo-director-ha-test-suite/test/test_instance-creation -r {{ working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_instance-creation -u > {{ working_dir }}/test_ha_instance.log 2>&1 +# Test: Instance deployment +- block: + - name: HA Test instance deploy on the overcloud (undercloud) + shell: > + {{ working_dir }}/tripleo-director-ha-test-suite/TD-ha-test-suite.sh -t {{ working_dir }}/tripleo-director-ha-test-suite/test/test_instance-creation -r {{ working_dir }}/tripleo-director-ha-test-suite/recovery/recovery_instance-creation -u + register: test_ha_instance_cmd + always: + - name: copy stdout test result to undercloud and check command + copy: content="{{ test_ha_instance_cmd.stdout }}" dest="{{ validate_ha_logs_dir }}/test_ha_instance_stdout.log" + rescue: + - name: copy stderr test result to undercloud and check command + copy: content="{{ test_ha_instance_cmd.stderr }}" dest="{{ validate_ha_logs_dir }}/test_ha_instance_stderr.log" when: test_ha_instance|bool