From 5c0373b28743f42cbff345532a498ab3c52c3d91 Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Thu, 18 Mar 2021 10:47:47 +0000 Subject: [PATCH] Run all main target programs' tests Run tests from all main target programs, for that a new refstack_target_programs var is defined. The var will cause the role to execute 'refstack-client test' command for every given target program. Collect whole .stestr dir in the zuul jobs as there will be multiple output files (for every given target program). Bump the default guideline to the latest one (2020.11). The patch also fixes a few style mistakes/typos. Change-Id: I82ba0bf26b994d4e21acabcb015d798081a33da2 --- .zuul.yaml | 3 +-- README.md | 3 ++- defaults/main.yaml | 6 +++++- infrared_plugin/plugin.spec | 7 ++++++- playbooks/refstack-client-devstack.yaml | 2 +- tasks/main.yaml | 21 ++++++++++++++++----- tasks/post-tasks.yaml | 13 ++++++++----- tasks/run-refstack-client.yaml | 5 +++-- 8 files changed, 42 insertions(+), 18 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 2c36179..830b4ac 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -31,8 +31,7 @@ zuul_copy_output: "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/etc/accounts.yaml": logs "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/etc/tempest.conf": logs - "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/1.json": logs - "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/1": logs + "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/": logs "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/tempest.log": logs run: playbooks/refstack-client-devstack.yaml irrelevant-files: diff --git a/README.md b/README.md index 07f65cf..410b017 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,10 @@ deployment is working by passing refstack tests. | guideline | False | 2020.06 | String | Specific refstack guideline. | | private_key_path_src * | False | None | String | If defined, the key defined by the param is copied to the targeted machine to private_key_path location.| | refstack_client_source | False | ~/.refstack-client | String | Destination where refstack-client will be cloned only if the dir doesn't exist or it's empty, otherwise it will be used as is.| -| tempestconf_source | False | None | String | A path to python-tempestconf's source. If not defined, refstack-client's default is applied. | +| tempestconf_source | False | None | String | A path to python-tempestconf's source. If not defined, refstack-client's default is applied. | | refstack_client_version | False | HEAD | String | Version of refstack-client cloned from git. | | server | False | https://refstack.openstack.org/api | String | Server url where the results will be uploaded to if upload_results is True. | +| refstack_target_programs | False | ['platform', 'compute', 'object'] | List | The tests within the specified target programs will be executed. | | tempest_account_concurrency | False | 3 | Int | A concurrency accounts.yaml file is generated with by tempest. | | tempest_config_path | False | None | String | Destination of tempest configuration file to be used for running refstack tests. | | tempest_tag | False | refstack-client's default | String | Tempest will be cloned and checkouted to this specific tag. | diff --git a/defaults/main.yaml b/defaults/main.yaml index 41a8404..37bdda7 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -16,7 +16,11 @@ download_artifacts: false dest_dir: "{{ lookup('env', 'PWD') }}" # the latest guideline by default -guideline: "2020.06" +guideline: "2020.11" +refstack_target_programs: + - platform + - compute + - object url_cirros_image: "http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img" # a concurrency accounts.yaml file is generated with by tempest tempest_account_concurrency: 3 diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index b04f9da..47b9157 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -80,7 +80,12 @@ subparsers: type: Value help: | A path or an URL to a test list text file containing specific - test cases.a + test cases. + refstack_target_programs: + type: ListValue + help: | + Tests within the specified target programs will be executed. + Usage: --refstack_target_programs platform,compute upload_results: type: Bool help: | diff --git a/playbooks/refstack-client-devstack.yaml b/playbooks/refstack-client-devstack.yaml index 7b1b274..1149b04 100644 --- a/playbooks/refstack-client-devstack.yaml +++ b/playbooks/refstack-client-devstack.yaml @@ -24,7 +24,7 @@ include_tasks: ../tasks/main.yaml vars: refstack_client_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}" - tempestconf_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/python-tempestconf']. src_dir }}" + tempestconf_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/python-tempestconf'].src_dir }}" source_credentials: "{{ devstack_base_dir }}/devstack/openrc demo demo" source_admin_credentials: "{{ devstack_base_dir }}/devstack/openrc admin admin" additional_tempestconf_params: "auth.tempest_roles Member object-storage.operator_role Member" diff --git a/tasks/main.yaml b/tasks/main.yaml index b1de0b4..23b9f1e 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -27,8 +27,19 @@ - include_tasks: post-tasks.yaml when: upload_results | bool or download_artifacts | bool -- name: Check if we passed refstack tests - fail: - msg: "Refstack tests failed." - # the refstack_result var got set in run-refstack-client.yaml - when: refstack_result.rc > 0 +- when: test_list is defined + block: + - name: Check if we passed refstack tests + fail: + msg: "Refstack tests failed." + # the refstack_result var got set in run-refstack-client.yaml + when: refstack_result.rc > 0 + +- when: test_list is not defined + block: + - name: Check if we passed tests of all specified target programs + fail: + msg: "Refstack tests failed." + # the refstack_result var got set in run-refstack-client.yaml + when: item.rc > 0 + with_items: "{{ refstack_result.results }}" diff --git a/tasks/post-tasks.yaml b/tasks/post-tasks.yaml index 78eb376..ead3f06 100644 --- a/tasks/post-tasks.yaml +++ b/tasks/post-tasks.yaml @@ -26,18 +26,19 @@ shell: | set -ex source .venv/bin/activate - refstack-client upload -y "{{ ls_out.files[-1].path }}" \ + refstack-client upload -y "{{ ls_out.files[-(item[0] | int + 1) | int].path }}" \ --url {{ server }} \ -i {{ private_key_path }} register: upload_out args: chdir: "{{ refstack_client_source }}" executable: /bin/bash + with_indexed_items: "{{ refstack_target_programs }}" when: - private_key_path is defined - debug: - msg: "{{ upload_out.stdout }}" + msg: "{{ upload_out }}" # this becomes handy when the role is executed on a remote node from # an executor node (e.g. a Jenkins job) when the below artifacts might be @@ -46,19 +47,21 @@ block: - name: Download results file in .json fetch: - src: "{{ ls_out.files[-1].path }}" + src: "{{ ls_out.files[-(item[0] | int + 1) | int].path }}" dest: "{{ dest_dir }}/test_results.json" flat: true + with_indexed_items: "{{ refstack_target_programs }}" - name: Download results file in subunit fetch: - src: "{{ ls_out.files[-1].path | splitext | first }}" + src: "{{ ls_out.files[-(item[0] | int + 1) | int].path | splitext | first }}" dest: "{{ dest_dir }}/test_results_subunit" flat: true + with_indexed_items: "{{ refstack_target_programs }}" - name: Dump output of upload command copy: - content: "{{ upload_out.stdout }}" + content: "{{ upload_out }}" dest: "{{ dest_dir }}/upload_output.txt" mode: '0644' delegate_to: localhost diff --git a/tasks/run-refstack-client.yaml b/tasks/run-refstack-client.yaml index 456b72a..6919292 100644 --- a/tasks/run-refstack-client.yaml +++ b/tasks/run-refstack-client.yaml @@ -32,7 +32,7 @@ ignore_errors: true when: test_list is defined -- name: Run tests associated with the specified guideline +- name: "Run {{ refstack_target_programs }} program tests associated within the {{ guideline }} guideline" shell: | set -ex export PATH=$PATH:/usr/local/sbin:/usr/sbin @@ -41,7 +41,7 @@ refstack-client test \ -c {{ path_to_tempest_config }} \ -v \ - --test-list "https://refstack.openstack.org/api/v1/guidelines/{{ guideline }}/tests?target=platform&type=required&alias=true&flag=false" + --test-list "https://refstack.openstack.org/api/v1/guidelines/{{ guideline }}/tests?target={{ item }}&type=required&alias=true&flag=false" args: chdir: "{{ refstack_client_source }}" executable: /bin/bash @@ -50,4 +50,5 @@ # the refstack_result is inspected in the last task of the role where it # fails if it failed here ignore_errors: true + with_items: "{{ refstack_target_programs }}" when: test_list is not defined