From 859924eeb42fec546234f62180ce213e3ce82852 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Fri, 2 Mar 2018 17:51:19 +0200 Subject: [PATCH] Various improvements for rally-task-at-devstack job * Move zuul configuration to .zuul.d directory. It allows to place job definitions into separate files (we have a lot of jobs). * Copy `extra` files to the right folder * Add check for rally results directory existance before fetching it * Stop play if `rally task start` fails with unexpected error code * Export RALLY_PLUGINS_DIR and RALLY_EXTRA_DIR environment variables before launching rally task, to provide backward compatibility. (see https://github.com/openstack/mistral/blob/master/rally-jobs/task-mistral.yaml#L1 ) * Save one line message about happened error * Load custom plugins Change-Id: I1004afb29b9c173e414c59ef54bf58458b5d9422 --- .zuul.d/README.rst | 33 +++++++++ .zuul.d/base.yaml | 19 +++++ .zuul.d/rally-task-heat.yaml | 22 ++++++ .zuul.d/rally-task-manila.yaml | 11 +++ .zuul.d/rally-task-nova.yaml | 5 ++ .zuul.d/rally-task-simple-job.yaml | 6 ++ .zuul.d/zuul.yaml | 52 ++++++++++++++ .zuul.yaml | 70 ------------------- .../roles/check-os-resources/tasks/main.yaml | 10 +++ .../fetch-rally-task-results/tasks/main.yaml | 32 ++++++--- .../prepare-for-rally-task/tasks/main.yaml | 10 +-- .../roles/run-rally-task/tasks/main.yaml | 40 +++++++++-- 12 files changed, 220 insertions(+), 90 deletions(-) create mode 100644 .zuul.d/README.rst create mode 100644 .zuul.d/base.yaml create mode 100644 .zuul.d/rally-task-heat.yaml create mode 100644 .zuul.d/rally-task-manila.yaml create mode 100644 .zuul.d/rally-task-nova.yaml create mode 100644 .zuul.d/rally-task-simple-job.yaml create mode 100644 .zuul.d/zuul.yaml delete mode 100644 .zuul.yaml diff --git a/.zuul.d/README.rst b/.zuul.d/README.rst new file mode 100644 index 00000000..470acb4b --- /dev/null +++ b/.zuul.d/README.rst @@ -0,0 +1,33 @@ +===================== +Zuul V3 configuration +===================== + +Zuul is a pipeline-oriented project gating system. It facilitates running +tests and automated tasks in response to Code Review events. + +See `official doc +`_ for more +details. + +What do we have in this this dir? +--------------------------------- + +.. note:: Do not document all files and jobs here. It will (for sure) become + outdated at some point. + +* **zuul.yaml** - the main configuration file. It contains a list of jobs + which should be launched at CI for rally-openstack project + +* **base.yaml** - the second by importance file. It contains basic parent + jobs. + +* All other files are named as like a job for which they include definition. + +Where are the actual job playbooks? +----------------------------------- + +Unfortunately, Zuul defines *zuul.d* (as like *.zuul.d*) as a directory for +project configuration and job definitions. + +Ansible roles, tasks cannot be here, so we placed them at *tests/ci/playbooks* +directory. diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml new file mode 100644 index 00000000..6486ae27 --- /dev/null +++ b/.zuul.d/base.yaml @@ -0,0 +1,19 @@ +- job: + name: rally-task-at-devstack + parent: devstack + description: Base job for launching Rally task at Devstack + required-projects: + - name: openstack-dev/devstack + - name: openstack-infra/devstack-gate + - name: openstack/rally-openstack + # NOTE(andreykurilin): it is a required project to fetch the latest + # version and test master of rally-openstack with master of rally + - name: openstack/rally + timeout: 7200 + roles: + - zuul: openstack-dev/devstack + vars: + devstack_plugins: + rally-openstack: https://git.openstack.org/openstack/rally-openstack + run: tests/ci/playbooks/run-rally-task.yaml + post-run: tests/ci/playbooks/post-rally-task.yaml diff --git a/.zuul.d/rally-task-heat.yaml b/.zuul.d/rally-task-heat.yaml new file mode 100644 index 00000000..e9506727 --- /dev/null +++ b/.zuul.d/rally-task-heat.yaml @@ -0,0 +1,22 @@ +- job: + name: rally-task-heat + parent: rally-task-at-devstack + vars: + devstack_plugins: + rally-openstack: https://git.openstack.org/openstack/rally-openstack + heat: https://git.openstack.org/openstack/heat + devstack_services: + rally-openstack: true + cinder: true + heat: true + # disable redundant services for the job + ceilometer-acentral: false + ceilometer-acompute: false + ceilometer-alarm-evaluator: false + ceilometer-alarm-notifier: false + ceilometer-anotification: false + ceilometer-api: false + ceilometer-collector: false + horizon: false + tempest: false + rally_task: rally-jobs/heat.yaml diff --git a/.zuul.d/rally-task-manila.yaml b/.zuul.d/rally-task-manila.yaml new file mode 100644 index 00000000..d33d4509 --- /dev/null +++ b/.zuul.d/rally-task-manila.yaml @@ -0,0 +1,11 @@ +- job: + name: rally-task-manila + parent: rally-task-at-devstack + vars: + rally_task: rally-jobs/rally-manila.yaml + +- job: + name: rally-task-manila-no-ss + parent: rally-task-at-devstack + vars: + rally_task: rally-jobs/manila-no-ss.yaml diff --git a/.zuul.d/rally-task-nova.yaml b/.zuul.d/rally-task-nova.yaml new file mode 100644 index 00000000..83422d81 --- /dev/null +++ b/.zuul.d/rally-task-nova.yaml @@ -0,0 +1,5 @@ +- job: + name: rally-task-nova + parent: rally-task-at-devstack + vars: + rally_task: rally-jobs/nova.yaml diff --git a/.zuul.d/rally-task-simple-job.yaml b/.zuul.d/rally-task-simple-job.yaml new file mode 100644 index 00000000..acd1f974 --- /dev/null +++ b/.zuul.d/rally-task-simple-job.yaml @@ -0,0 +1,6 @@ +- job: + name: rally-task-simple-job + description: A simple job which checks Rally integration with Devstack in Zuul V3 env. + parent: rally-task-at-devstack + vars: + rally_task: rally-jobs/simple-cinder.yaml diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml new file mode 100644 index 00000000..9e54efcf --- /dev/null +++ b/.zuul.d/zuul.yaml @@ -0,0 +1,52 @@ +- job: + name: rally-tox-pep8 + parent: tox + description: | + Run test for rally project. + + Uses tox with the ``pep8`` environment. + vars: + tox_envlist: pep8 + +- job: + name: rally-tox-py27 + parent: tox + description: | + Run test for rally project. + + Uses tox with the ``py27`` environment. + vars: + tox_envlist: py27 + +- job: + name: rally-tox-py35 + parent: tox + description: | + Run test for rally project. + + Uses tox with the ``py35`` environment. + vars: + tox_envlist: py35 + +- project: + check: + jobs: + - rally-tox-pep8 + - rally-tox-py27 + - rally-tox-py35 + - rally-task-simple-job + # it should fail at installing devstack step + - rally-task-heat: + voting: false + # it should not pass SLA + - rally-task-nova: + voting: false + # should fail at validation step + - rally-task-manila: + voting: false + gate: + jobs: + - rally-tox-pep8 + - rally-tox-py27 + - rally-tox-py35 + - rally-task-simple-job diff --git a/.zuul.yaml b/.zuul.yaml deleted file mode 100644 index 6627712e..00000000 --- a/.zuul.yaml +++ /dev/null @@ -1,70 +0,0 @@ -- job: - name: rally-tox-pep8 - parent: tox - description: | - Run test for rally project. - - Uses tox with the ``pep8`` environment. - vars: - tox_envlist: pep8 - -- job: - name: rally-tox-py27 - parent: tox - description: | - Run test for rally project. - - Uses tox with the ``py27`` environment. - vars: - tox_envlist: py27 - -- job: - name: rally-tox-py35 - parent: tox - description: | - Run test for rally project. - - Uses tox with the ``py35`` environment. - vars: - tox_envlist: py35 - -- job: - name: rally-task-at-devstack - parent: devstack - description: Base job for launching Rally task at Devstack - required-projects: - - name: openstack-dev/devstack - - name: openstack-infra/devstack-gate - - name: openstack/rally-openstack - # NOTE(andreykurilin): it is a required project to fetch the latest - # version and test master of rally-openstack with master of rally - - name: openstack/rally - timeout: 7200 - roles: - - zuul: openstack-dev/devstack - vars: - devstack_plugins: - rally-openstack: https://git.openstack.org/openstack/rally-openstack - run: tests/ci/playbooks/run-rally-task.yaml - post-run: tests/ci/playbooks/post-rally-task.yaml - -- job: - name: rally-task-simple-job - parent: rally-task-at-devstack - vars: - rally_task: rally-jobs/simple-cinder.yaml - -- project: - check: - jobs: - - rally-tox-pep8 - - rally-tox-py27 - - rally-tox-py35 - - rally-task-simple-job - gate: - jobs: - - rally-tox-pep8 - - rally-tox-py27 - - rally-tox-py35 - - rally-task-simple-job - diff --git a/tests/ci/playbooks/roles/check-os-resources/tasks/main.yaml b/tests/ci/playbooks/roles/check-os-resources/tasks/main.yaml index f3950a29..09a3ec8e 100644 --- a/tests/ci/playbooks/roles/check-os-resources/tasks/main.yaml +++ b/tests/ci/playbooks/roles/check-os-resources/tasks/main.yaml @@ -4,4 +4,14 @@ shell: "python ./tests/ci/osresources.py --compare-with-list {{ rally_resources_at_start }}" args: chdir: "{{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}" + register: command_result + ignore_errors: True +- name: Check results of listing resources + shell: + cmd: | + if [ "{{ command_result.rc }}" != "0" ] && [ ! -f "{{ rally_results_dir }}/status.txt" ]; then + echo "OSResources check found resources which appeared after Rally Task execution. Check cleanup logs for errors." > "{{ rally_results_dir }}/status.txt" + fi + + exit "{{ command_result.rc }}" diff --git a/tests/ci/playbooks/roles/fetch-rally-task-results/tasks/main.yaml b/tests/ci/playbooks/roles/fetch-rally-task-results/tasks/main.yaml index dc0a979d..d9b38cf2 100644 --- a/tests/ci/playbooks/roles/fetch-rally-task-results/tasks/main.yaml +++ b/tests/ci/playbooks/roles/fetch-rally-task-results/tasks/main.yaml @@ -1,9 +1,30 @@ +- name: Change the status message + become: yes + shell: + cmd: | + set -e + if [ -f "{{ rally_results_dir }}/status.txt" ]; then + MSG=`cat {{ rally_results_dir }}/status.txt` + else + MSG="Unexpected error had happened before Rally Task start. Check ARA for more details." + fi + + if [ "$MSG" != "OK" ]; then + sed -i -- "s/var errorMessage = null\;/var errorMessage = \'$MSG\';/g" {{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}/tests/ci/pages/task-index.html + fi + +- name: Check the existence of results dir + stat: + path: '{{ rally_results_dir }}' + register: results_dir_stat + - name: Gzip results files become: yes become_user: stack - shell: "gzip * -9 -q" + shell: "gzip * -9 -q | true" args: chdir: '{{ rally_results_dir }}' + when: results_dir_stat.stat.exists == True - name: Gzip custom plugins become: yes @@ -11,6 +32,7 @@ shell: "tar -czf {{ rally_results_dir }}/plugins.tar.gz ./plugins" args: chdir: '{{ rally_home_dir }}' + when: results_dir_stat.stat.exists == True - name: Copy Rally results become: yes @@ -25,13 +47,7 @@ - --include=*/ - --exclude=* - --prune-empty-dirs - -- name: Change the status message - become: yes - shell: - cmd: | - set -e - #sed -i -- "s/var errorMessage = null\;/var errorMessage = \'TBD\';/g" {{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}/tests/ci/pages/task-index.html + when: results_dir_stat.stat.exists == True - name: Put index.html in the right place become: yes diff --git a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml index cb9c2593..05a86da7 100644 --- a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml +++ b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml @@ -53,7 +53,7 @@ - name: Copy extra files, if they are presented become: True become_user: stack - command: cp -r {{ zuul.project.src_dir }}/rally-jobs/extra {{ rally_home_dir }}/extra + command: cp -r {{ zuul.project.src_dir }}/rally-jobs/extra {{ rally_home_dir }}/ when: extra_files_stat.stat.exists == True - name: Check the existence of fake image @@ -142,7 +142,7 @@ - name: Create new projects and users become: True become_user: stack - shell: rally env create --name devstask-with-users --spec {{ rally_existing_users_config }} + shell: rally env create --name devstask-with-users --spec "{{ rally_existing_users_config }}" when: rally_use_existing_users == True - name: Print Rally deployment config @@ -178,7 +178,7 @@ - name: Copy task file become: True become_user: stack - command: cp -r {{ zuul.project.src_dir }}/{{ rally_task }} {{ rally_home_dir }}/task.yaml + command: cp "{{ zuul.project.src_dir }}/{{ rally_task }}" "{{ rally_home_dir }}/task.yaml" - name: Check the existence of task_args_file stat: @@ -188,11 +188,11 @@ - name: Copy task_args_file become: True become_user: stack - command: cp {{ zuul.project.src_dir }}/{{ rally_task_args_file }} {{ rally_home_dir }}/task_args_file.yaml + command: cp "{{ zuul.project.src_dir }}/{{ rally_task_args_file }}" "{{ rally_home_dir }}/task_args_file.yaml" when: task_args_file_stat.stat.exists == True - name: Create an empty task_args_file become: True become_user: stack - command: echo "{}" > {{ rally_home_dir }}/task_args_file.yaml + command: echo "{}" > "{{ rally_home_dir }}/task_args_file.yaml" when: task_args_file_stat.stat.exists == False diff --git a/tests/ci/playbooks/roles/run-rally-task/tasks/main.yaml b/tests/ci/playbooks/roles/run-rally-task/tasks/main.yaml index b49c16a0..a77b033e 100644 --- a/tests/ci/playbooks/roles/run-rally-task/tasks/main.yaml +++ b/tests/ci/playbooks/roles/run-rally-task/tasks/main.yaml @@ -3,12 +3,38 @@ become_user: stack shell: cmd: | - set +e - rally --rally-debug task start --task {{ rally_home_dir }}/task.yaml #--task-args-file {{ rally_home_dir }}/task_args_file.yaml - retval=$? - if [[ $? -eq 1 ]]; then - # 1 exit code means that something unexpected had happened - exit $retval - fi + export RALLY_PLUGINS_DIR="{{ rally_home_dir }}/plugins" + export RALLY_EXTRA_DIR="{{ rally_home_dir }}/extra" + + rally --rally-debug --plugin-paths "{{ rally_home_dir }}/plugins" task start --task {{ rally_home_dir }}/task.yaml #--task-args-file {{ rally_home_dir }}/task_args_file.yaml executable: /bin/bash + register: command_result + ignore_errors: True + +- name: Check rally task exit code + become: True + become_user: stack + shell: + cmd: | + STATUS="OK" + EXIT_CODE=0 + if [ "{{ command_result.rc }}" -eq "2" ]; then + STATUS="At least one workload did not pass SLA criteria." + EXIT_CODE=0 + # actually, InvalidTaskException has 457 error code, but exit codes are + # limited by the range 0-255 . So 457 means 201... need to fix exit + # codes + elif [ "{{ command_result.rc }} " -eq "201" ]; then + STATUS="Task config is invalid. Check logs for validation errors." + # no results can be produced. stop play + EXIT_CODE=1 + elif [ "{{ command_result.rc }}" -ne "0" ]; then + STATUS="Unexpected error had happened while launching Rally task. Check logs for more details." + EXIT_CODE=1 + fi + + # dump status, so it can be parsed in post script + echo "$STATUS" > "{{ rally_results_dir }}/status.txt" + + exit $EXIT_CODE