Various improvements for rally-task-at-devstack job

* Move zuul configuration to .zuul.d directory. It allows to place job
  definitions into separate files (we have a lot of jobs).
* Copy `extra` files to the right folder
* Add check for rally results directory existance before fetching it
* Stop play if `rally task start` fails with unexpected error code
* Export RALLY_PLUGINS_DIR and RALLY_EXTRA_DIR environment variables
  before launching rally task, to provide backward compatibility.
  (see https://github.com/openstack/mistral/blob/master/rally-jobs/task-mistral.yaml#L1 )
* Save one line message about happened error
* Load custom plugins

Change-Id: I1004afb29b9c173e414c59ef54bf58458b5d9422

.zuul.d/README.rst

@@ -0,0 +1,33 @@
+=====================
+Zuul V3 configuration
+=====================
+
+Zuul is a pipeline-oriented project gating system. It facilitates running
+tests and automated tasks in response to Code Review events.
+
+See `official doc
+<https://docs.openstack.org/infra/system-config/zuulv3.html>`_ for more
+details.
+
+What do we have in this this dir?
+---------------------------------
+
+.. note:: Do not document all files and jobs here. It will (for sure) become
+    outdated at some point.
+
+* **zuul.yaml** - the main configuration file. It contains a list of jobs
+  which should be launched at CI for rally-openstack project
+
+* **base.yaml** - the second by importance file. It contains basic parent
+  jobs.
+
+* All other files are named as like a job for which they include definition.
+
+Where are the actual job playbooks?
+-----------------------------------
+
+Unfortunately, Zuul defines *zuul.d* (as like *.zuul.d*) as a directory for
+project configuration and job definitions.
+
+Ansible roles, tasks cannot be here, so we placed them at *tests/ci/playbooks*
+directory.

.zuul.d/base.yaml

@@ -0,0 +1,19 @@
+- job:
+    name: rally-task-at-devstack
+    parent: devstack
+    description: Base job for launching Rally task at Devstack
+    required-projects:
+      - name: openstack-dev/devstack
+      - name: openstack-infra/devstack-gate
+      - name: openstack/rally-openstack
+      # NOTE(andreykurilin): it is a required project to fetch the latest
+      #   version and test master of rally-openstack with master of rally
+      - name: openstack/rally
+    timeout: 7200
+    roles:
+      - zuul: openstack-dev/devstack
+    vars:
+      devstack_plugins:
+        rally-openstack: https://git.openstack.org/openstack/rally-openstack
+    run: tests/ci/playbooks/run-rally-task.yaml
+    post-run: tests/ci/playbooks/post-rally-task.yaml

.zuul.d/rally-task-heat.yaml

@@ -0,0 +1,22 @@
+- job:
+    name: rally-task-heat
+    parent: rally-task-at-devstack
+    vars:
+      devstack_plugins:
+        rally-openstack: https://git.openstack.org/openstack/rally-openstack
+        heat: https://git.openstack.org/openstack/heat
+      devstack_services:
+        rally-openstack: true
+        cinder: true
+        heat: true
+        # disable redundant services for the job
+        ceilometer-acentral: false
+        ceilometer-acompute: false
+        ceilometer-alarm-evaluator: false
+        ceilometer-alarm-notifier: false
+        ceilometer-anotification: false
+        ceilometer-api: false
+        ceilometer-collector: false
+        horizon: false
+        tempest: false
+      rally_task: rally-jobs/heat.yaml

.zuul.d/rally-task-manila.yaml

@@ -0,0 +1,11 @@
+- job:
+    name: rally-task-manila
+    parent: rally-task-at-devstack
+    vars:
+      rally_task: rally-jobs/rally-manila.yaml
+
+- job:
+    name: rally-task-manila-no-ss
+    parent: rally-task-at-devstack
+    vars:
+      rally_task: rally-jobs/manila-no-ss.yaml

.zuul.d/rally-task-nova.yaml

@@ -0,0 +1,5 @@
+- job:
+    name: rally-task-nova
+    parent: rally-task-at-devstack
+    vars:
+      rally_task: rally-jobs/nova.yaml

.zuul.d/rally-task-simple-job.yaml

@@ -0,0 +1,6 @@
+- job:
+    name: rally-task-simple-job
+    description: A simple job which checks Rally integration with Devstack in Zuul V3 env.
+    parent: rally-task-at-devstack
+    vars:
+      rally_task: rally-jobs/simple-cinder.yaml

.zuul.d/zuul.yaml

@@ -0,0 +1,52 @@
+- job:
+    name: rally-tox-pep8
+    parent: tox
+    description: |
+      Run test for rally project.
+
+      Uses tox with the ``pep8`` environment.
+    vars:
+      tox_envlist: pep8
+
+- job:
+    name: rally-tox-py27
+    parent: tox
+    description: |
+      Run test for rally project.
+
+      Uses tox with the ``py27`` environment.
+    vars:
+      tox_envlist: py27
+
+- job:
+    name: rally-tox-py35
+    parent: tox
+    description: |
+      Run test for rally project.
+
+      Uses tox with the ``py35`` environment.
+    vars:
+      tox_envlist: py35
+
+- project:
+    check:
+      jobs:
+        - rally-tox-pep8
+        - rally-tox-py27
+        - rally-tox-py35
+        - rally-task-simple-job
+        # it should fail at installing devstack step
+        - rally-task-heat:
+            voting: false
+        # it should not pass SLA
+        - rally-task-nova:
+            voting: false
+        # should fail at validation step
+        - rally-task-manila:
+            voting: false
+    gate:
+      jobs:
+        - rally-tox-pep8
+        - rally-tox-py27
+        - rally-tox-py35
+        - rally-task-simple-job

.zuul.yaml

@@ -1,70 +0,0 @@
-- job:
-    name: rally-tox-pep8
-    parent: tox
-    description: |
-      Run test for rally project.
-
-      Uses tox with the ``pep8`` environment.
-    vars:
-      tox_envlist: pep8
-
-- job:
-    name: rally-tox-py27
-    parent: tox
-    description: |
-      Run test for rally project.
-
-      Uses tox with the ``py27`` environment.
-    vars:
-      tox_envlist: py27
-
-- job:
-    name: rally-tox-py35
-    parent: tox
-    description: |
-      Run test for rally project.
-
-      Uses tox with the ``py35`` environment.
-    vars:
-      tox_envlist: py35
-
-- job:
-    name: rally-task-at-devstack
-    parent: devstack
-    description: Base job for launching Rally task at Devstack
-    required-projects:
-      - name: openstack-dev/devstack
-      - name: openstack-infra/devstack-gate
-      - name: openstack/rally-openstack
-      # NOTE(andreykurilin): it is a required project to fetch the latest
-      #   version and test master of rally-openstack with master of rally
-      - name: openstack/rally
-    timeout: 7200
-    roles:
-      - zuul: openstack-dev/devstack
-    vars:
-      devstack_plugins:
-        rally-openstack: https://git.openstack.org/openstack/rally-openstack
-    run: tests/ci/playbooks/run-rally-task.yaml
-    post-run: tests/ci/playbooks/post-rally-task.yaml
-
-- job:
-    name: rally-task-simple-job
-    parent: rally-task-at-devstack
-    vars:
-      rally_task: rally-jobs/simple-cinder.yaml
-
-- project:
-    check:
-      jobs:
-        - rally-tox-pep8
-        - rally-tox-py27
-        - rally-tox-py35
-        - rally-task-simple-job
-    gate:
-      jobs:
-        - rally-tox-pep8
-        - rally-tox-py27
-        - rally-tox-py35
-        - rally-task-simple-job
-

tests/ci/playbooks/roles/check-os-resources/tasks/main.yaml

@@ -4,4 +4,14 @@
   shell: "python ./tests/ci/osresources.py --compare-with-list {{ rally_resources_at_start }}"
   args:
     chdir: "{{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}"
+  register: command_result
+  ignore_errors: True
 
+- name: Check results of listing resources
+  shell:
+    cmd: |
+      if [ "{{ command_result.rc }}" != "0" ] && [ ! -f "{{ rally_results_dir }}/status.txt" ]; then
+          echo "OSResources check found resources which appeared after Rally Task execution. Check cleanup logs for errors." > "{{ rally_results_dir }}/status.txt"
+      fi
+
+      exit "{{ command_result.rc }}"

tests/ci/playbooks/roles/fetch-rally-task-results/tasks/main.yaml

@@ -1,9 +1,30 @@
+- name: Change the status message
+  become: yes
+  shell:
+    cmd: |
+      set -e
+      if [ -f "{{ rally_results_dir }}/status.txt" ]; then
+        MSG=`cat {{ rally_results_dir }}/status.txt`
+      else
+        MSG="Unexpected error had happened before Rally Task start. Check ARA for more details."
+      fi
+
+      if [ "$MSG" != "OK" ]; then
+        sed -i -- "s/var errorMessage = null\;/var errorMessage = \'$MSG\';/g" {{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}/tests/ci/pages/task-index.html
+      fi
+
+- name: Check the existence of results dir
+  stat:
+    path: '{{ rally_results_dir }}'
+  register: results_dir_stat
+
 - name: Gzip results files
   become: yes
   become_user: stack
-  shell: "gzip * -9 -q"
+  shell: "gzip * -9 -q | true"
   args:
     chdir: '{{ rally_results_dir }}'
+  when: results_dir_stat.stat.exists == True
 
 - name: Gzip custom plugins
   become: yes
@@ -11,6 +32,7 @@
   shell: "tar -czf {{ rally_results_dir }}/plugins.tar.gz ./plugins"
   args:
     chdir: '{{ rally_home_dir }}'
+  when: results_dir_stat.stat.exists == True
 
 - name: Copy Rally results
   become: yes
@@ -25,13 +47,7 @@
       - --include=*/
       - --exclude=*
       - --prune-empty-dirs
-
-- name: Change the status message
-  become: yes
-  shell:
-    cmd: |
-       set -e
-       #sed -i -- "s/var errorMessage = null\;/var errorMessage = \'TBD\';/g" {{ zuul.projects['git.openstack.org/openstack/rally-openstack'].src_dir }}/tests/ci/pages/task-index.html
+  when: results_dir_stat.stat.exists == True
 
 - name: Put index.html in the right place
   become: yes

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml

@@ -53,7 +53,7 @@
 - name: Copy extra files, if they are presented
   become: True
   become_user: stack
-  command: cp -r {{ zuul.project.src_dir }}/rally-jobs/extra {{ rally_home_dir }}/extra
+  command: cp -r {{ zuul.project.src_dir }}/rally-jobs/extra {{ rally_home_dir }}/
   when: extra_files_stat.stat.exists == True
 
 - name: Check the existence of fake image
@@ -142,7 +142,7 @@
 - name: Create new projects and users
   become: True
   become_user: stack
-  shell: rally env create --name devstask-with-users --spec {{ rally_existing_users_config }}
+  shell: rally env create --name devstask-with-users --spec "{{ rally_existing_users_config }}"
   when: rally_use_existing_users == True
 
 - name: Print Rally deployment config
@@ -178,7 +178,7 @@
 - name: Copy task file
   become: True
   become_user: stack
-  command: cp -r {{ zuul.project.src_dir }}/{{ rally_task }} {{ rally_home_dir }}/task.yaml
+  command: cp "{{ zuul.project.src_dir }}/{{ rally_task }}" "{{ rally_home_dir }}/task.yaml"
 
 - name: Check the existence of task_args_file
   stat:
@@ -188,11 +188,11 @@
 - name: Copy task_args_file
   become: True
   become_user: stack
-  command: cp {{ zuul.project.src_dir }}/{{ rally_task_args_file }} {{ rally_home_dir }}/task_args_file.yaml
+  command: cp "{{ zuul.project.src_dir }}/{{ rally_task_args_file }}" "{{ rally_home_dir }}/task_args_file.yaml"
   when: task_args_file_stat.stat.exists == True
 
 - name: Create an empty task_args_file
   become: True
   become_user: stack
-  command: echo "{}" > {{ rally_home_dir }}/task_args_file.yaml
+  command: echo "{}" > "{{ rally_home_dir }}/task_args_file.yaml"
   when: task_args_file_stat.stat.exists == False

tests/ci/playbooks/roles/run-rally-task/tasks/main.yaml

@@ -3,12 +3,38 @@
   become_user: stack
   shell:
     cmd: |
-      set +e
 
-      rally --rally-debug task start --task {{ rally_home_dir }}/task.yaml #--task-args-file {{ rally_home_dir }}/task_args_file.yaml
-      retval=$?
-      if [[ $? -eq 1 ]]; then
-          # 1 exit code means that something unexpected had happened
-          exit $retval
-      fi
+      export RALLY_PLUGINS_DIR="{{ rally_home_dir }}/plugins"
+      export RALLY_EXTRA_DIR="{{ rally_home_dir }}/extra"
+
+      rally --rally-debug --plugin-paths "{{ rally_home_dir }}/plugins" task start --task {{ rally_home_dir }}/task.yaml #--task-args-file {{ rally_home_dir }}/task_args_file.yaml
     executable: /bin/bash
+  register: command_result
+  ignore_errors: True
+
+- name: Check rally task exit code
+  become: True
+  become_user: stack
+  shell:
+    cmd: |
+      STATUS="OK"
+      EXIT_CODE=0
+      if [ "{{ command_result.rc }}" -eq "2" ]; then
+          STATUS="At least one workload did not pass SLA criteria."
+          EXIT_CODE=0
+      # actually, InvalidTaskException has 457 error code, but exit codes are
+      # limited by the range 0-255 . So 457 means 201... need to fix exit
+      # codes
+      elif [ "{{ command_result.rc }} " -eq "201" ]; then
+          STATUS="Task config is invalid. Check logs for validation errors."
+          # no results can be produced. stop play
+          EXIT_CODE=1
+      elif [ "{{ command_result.rc }}" -ne "0" ]; then
+          STATUS="Unexpected error had happened while launching Rally task. Check logs for more details."
+          EXIT_CODE=1
+      fi
+
+      # dump status, so it can be parsed in post script
+      echo "$STATUS" > "{{ rally_results_dir }}/status.txt"
+
+      exit $EXIT_CODE