Rename rally-scenarios to rally-jobs & add readme

This patch renames misleading rally-scenarios to rally-jobs.
rally-jobs describes much better purpose of this directory

Add readme files that describes structure and meanings of files
and directories

Change-Id: I8f5a2816186a2cafd251981ba7ad50e631e548f5
Closes-bug: #1395502

rally-jobs/README.rst

@@ -0,0 +1,40 @@
+Rally job related files
+=======================
+
+This directory contains rally tasks and plugins that are run by OpenStack CI.
+
+Structure
+---------
+
+* plugins - directory where you can add rally plugins. Almost everything in
+  Rally is a plugin. Benchmark context, Benchmark scenario, SLA checks, Generic
+  cleanup resources, ....
+
+* extra - all files from this directory will be copy pasted to gates, so you
+  are able to use absolute paths in rally tasks.
+  Files will be located in ~/.rally/extra/*
+
+* rally.yaml is a task that is run in gates against OpenStack (nova network)
+
+* rally-neutron.yaml is a task that is run in gates against OpenStack with
+  Neutron Service
+
+* rally-designate.yaml is a task that is run in gates against OpenStack with
+  Designate Service. It's experimental job. To trigger make a review with
+  "check experimental" text.
+
+* rally-zaqar.yaml is a task that is run in gates against OpenStack with
+  Zaqar Service. It's experimental job. To trigger make a review with
+  "check experimental" text.
+
+
+Useful links
+------------
+
+* More about Rally: https://rally.readthedocs.org/en/latest/
+
+* How to add rally-gates: https://rally.readthedocs.org/en/latest/rally_gatejob.html
+
+* About plugins:  https://rally.readthedocs.org/en/latest/plugins.html
+
+* Plugin samples: https://github.com/stackforge/rally/tree/master/doc/samples/plugins

rally-jobs/extra/README.rst

@@ -0,0 +1,6 @@
+Extra files
+===========
+
+All files from this directory will be copy pasted to gates, so you are able to
+use absolute path in rally tasks. Files will be in ~/.rally/extra/*
+

rally-jobs/plugins/README.rst

@@ -0,0 +1,9 @@
+Rally plugins
+=============
+
+All *.py modules from this directory will be auto-loaded by Rally and all
+plugins will be discoverable. There is no need of any extra configuration
+and there is no difference between writing them here and in rally code base.
+
+Note that it is better to push all interesting and useful benchmarks to Rally
+code base, this simplifies administration for Operators.

tests/ci/rally-gate.sh

@@ -15,9 +15,15 @@
 # This script is executed by post_test_hook function in desvstack gate.
 
 PROJECT=`echo $ZUUL_PROJECT | cut -d \/ -f 2`
-SCENARIO=$BASE/new/$PROJECT/rally-scenarios/${RALLY_SCENARIO}.yaml
+
-PLUGINS_DIR=$BASE/new/$PROJECT/rally-scenarios/plugins
+RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-scenarios
-EXTRA_DIR=$BASE/new/$PROJECT/rally-scenarios/extra
+if [ ! -d $RALLY_JOB_DIR ]; then
+    RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs
+fi
+
+SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml
+PLUGINS_DIR=${RALLY_JOB_DIR}/plugins
+EXTRA_DIR=${RALLY_JOB_DIR}/extra
 
 RALLY_PLUGINS_DIR=~/.rally/plugins
 

tests/unit/rally_jobs/test_jobs.py

@@ -23,17 +23,18 @@ import rally.utils as rutils
 from tests.unit import test
 
 
-class ScenarioTestCase(test.TestCase):
+class RallyJobsTestCase(test.TestCase):
-    rally_scenarios_path = os.path.join(
+    rally_jobs_path = os.path.join(
-        os.path.dirname(__file__), "..", "..", "..", "rally-scenarios")
+        os.path.dirname(__file__), "..", "..", "..", "rally-jobs")
 
     @mock.patch("rally.benchmark.engine.BenchmarkEngine"
                 "._validate_config_semantic")
     def test_schema_is_valid(self, mock_validate):
-        rutils.load_plugins(os.path.join(self.rally_scenarios_path, "plugins"))
+        rutils.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
 
-        for filename in ["rally.yaml", "rally-neutron.yaml"]:
+        for filename in ["rally.yaml", "rally-neutron.yaml",
-            full_path = os.path.join(self.rally_scenarios_path, filename)
+                         "rally-zaqar.yaml", "rally-designate.yaml"]:
+            full_path = os.path.join(self.rally_jobs_path, filename)
 
             with open(full_path) as task_file:
                 try:
@@ -43,4 +44,4 @@ class ScenarioTestCase(test.TestCase):
                     eng.validate()
                 except Exception:
                     print(traceback.format_exc())
-                    self.fail("Wrong scenario config %s" % full_path)
+                    self.fail("Wrong task input file: %s" % full_path)