Rename rally-scenarios to rally-jobs & add readme

This patch renames misleading rally-scenarios to rally-jobs.
rally-jobs describes much better purpose of this directory

Add readme files that describes structure and meanings of files
and directories

Change-Id: I8f5a2816186a2cafd251981ba7ad50e631e548f5
Closes-bug: #1395502

rally-jobs/README.rst

@@ -0,0 +1,40 @@
+Rally job related files
+=======================
+
+This directory contains rally tasks and plugins that are run by OpenStack CI.
+
+Structure
+---------
+
+* plugins - directory where you can add rally plugins. Almost everything in
+  Rally is a plugin. Benchmark context, Benchmark scenario, SLA checks, Generic
+  cleanup resources, ....
+
+* extra - all files from this directory will be copy pasted to gates, so you
+  are able to use absolute paths in rally tasks.
+  Files will be located in ~/.rally/extra/*
+
+* rally.yaml is a task that is run in gates against OpenStack (nova network)
+
+* rally-neutron.yaml is a task that is run in gates against OpenStack with
+  Neutron Service
+
+* rally-designate.yaml is a task that is run in gates against OpenStack with
+  Designate Service. It's experimental job. To trigger make a review with
+  "check experimental" text.
+
+* rally-zaqar.yaml is a task that is run in gates against OpenStack with
+  Zaqar Service. It's experimental job. To trigger make a review with
+  "check experimental" text.
+
+
+Useful links
+------------
+
+* More about Rally: https://rally.readthedocs.org/en/latest/
+
+* How to add rally-gates: https://rally.readthedocs.org/en/latest/rally_gatejob.html
+
+* About plugins:  https://rally.readthedocs.org/en/latest/plugins.html
+
+* Plugin samples: https://github.com/stackforge/rally/tree/master/doc/samples/plugins

rally-jobs/extra/README.rst

@@ -0,0 +1,6 @@
+Extra files
+===========
+
+All files from this directory will be copy pasted to gates, so you are able to
+use absolute path in rally tasks. Files will be in ~/.rally/extra/*
+

rally-jobs/extra/server_with_ports.yaml.template

@@ -0,0 +1,53 @@
+heat_template_version: 2013-05-23
+
+parameters:
+  # set all correct defaults for parameters before launch test
+  public_net:
+    type: string
+    default: public
+  image:
+    type: string
+    default: cirros-0.3.2-x86_64-uec
+  flavor:
+    type: string
+    default: m1.tiny
+  cidr:
+    type: string
+    default: 11.11.11.0/24
+
+resources:
+  server:
+    type: OS::Nova::Server
+    properties:
+      image: {get_param: image}
+      flavor: {get_param: flavor}
+      networks:
+        - port: { get_resource: server_port }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: {get_param: public_net}
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+  private_net:
+    type: OS::Neutron::Net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network: { get_resource: private_net }
+      cidr: {get_param: cidr}
+
+  server_port:
+    type: OS::Neutron::Port
+    properties:
+      network: {get_resource: private_net}
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }

rally-jobs/extra/server_with_volume.yaml.template

@@ -0,0 +1,39 @@
+heat_template_version: 2013-05-23
+
+parameters:
+  # set all correct defaults for parameters before launch test
+  image:
+    type: string
+    default: cirros-0.3.2-x86_64-uec
+  flavor:
+    type: string
+    default: m1.tiny
+  availability_zone:
+    type: string
+    description: The Availability Zone to launch the instance.
+    default: nova
+  volume_size:
+    type: number
+    description: Size of the volume to be created.
+    default: 1
+    constraints:
+      - range: { min: 1, max: 1024 }
+        description: must be between 1 and 1024 Gb.
+
+resources:
+  server:
+    type: OS::Nova::Server
+    properties:
+      image: {get_param: image}
+      flavor: {get_param: flavor}
+  cinder_volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: { get_param: volume_size }
+      availability_zone: { get_param: availability_zone }
+  volume_attachment:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      volume_id: { get_resource: cinder_volume }
+      instance_uuid: { get_resource: server}
+      mountpoint: /dev/vdc

rally-jobs/plugins/README.rst

@@ -0,0 +1,9 @@
+Rally plugins
+=============
+
+All *.py modules from this directory will be auto-loaded by Rally and all
+plugins will be discoverable. There is no need of any extra configuration
+and there is no difference between writing them here and in rally code base.
+
+Note that it is better to push all interesting and useful benchmarks to Rally
+code base, this simplifies administration for Operators.

rally-jobs/plugins/fake_plugin.py

@@ -0,0 +1,36 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import random
+import time
+
+from rally.benchmark.scenarios import base
+
+
+class FakePlugin(base.Scenario):
+
+    @base.atomic_action_timer("test1")
+    def _test1(self, factor):
+        time.sleep(random.random())
+
+    @base.atomic_action_timer("test2")
+    def _test2(self, factor):
+        time.sleep(random.random() * factor * 10)
+
+    @base.scenario()
+    def testplugin(self, factor=1):
+        self._test1(factor)
+        self._test2(factor)

rally-jobs/rally-designate.yaml

@@ -0,0 +1,69 @@
+---
+  DesignateBasic.create_and_delete_domain:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 10
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  DesignateBasic.create_and_delete_records:
+    -
+      args:
+        records_per_domain: 10
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 10
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  DesignateBasic.create_and_list_domains:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 10
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  DesignateBasic.create_and_list_records:
+    -
+      args:
+        records_per_domain: 10
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 10
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  DesignateBasic.list_domains:
+    -
+      runner:
+        type: "constant"
+        times: 3
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0

rally-jobs/rally-neutron.yaml

@@ -0,0 +1,459 @@
+---
+  Dummy.dummy:
+    -
+      args:
+        sleep: 0.01
+      runner:
+        type: "constant"
+        times: 1
+        concurrency: 1
+      context:
+        users:
+          tenants: 5
+          users_per_tenant: 5
+        quotas:
+          nova:
+            instances: 200
+            cores: 200
+            ram: -1
+            metadata_items: -1
+            injected_files: -1
+            injected_file_content_bytes: -1
+            injected_file_path_bytes: -1
+            key_pairs: 500
+          cinder:
+            gigabytes: -1
+            snapshots: -1
+            volumes: -1
+          neutron:
+            network: -1
+            subnet: -1
+            port: 200
+            router: 300
+            floatingip: -1
+            security_group: -1
+            security_group_rule: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_list_networks:
+    -
+      args:
+        network_create_args:
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+             network: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_list_subnets:
+    -
+      args:
+        network_create_args:
+        subnet_create_args:
+        subnet_cidr_start: "1.1.0.0/30"
+        subnets_per_network: 2
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+             network: -1
+             subnet: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_list_routers:
+    -
+      args:
+        network_create_args:
+        subnet_create_args:
+        subnet_cidr_start: "1.1.0.0/30"
+        subnets_per_network: 2
+        router_create_args:
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+             network: -1
+             subnet: -1
+             router: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_list_ports:
+    -
+      args:
+        network_create_args:
+        port_create_args:
+        ports_per_network: 5
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+             network: -1
+             subnet: -1
+             router: -1
+             port: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_update_networks:
+    -
+      args:
+        network_create_args: {}
+        network_update_args:
+            admin_state_up: False
+            name: "_updated"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 5
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_update_subnets:
+    -
+      args:
+        network_create_args: {}
+        subnet_create_args: {}
+        subnet_cidr_start: "1.4.0.0/16"
+        subnets_per_network: 2
+        subnet_update_args:
+            enable_dhcp: False
+            name: "_subnet_updated"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 5
+      context:
+        users:
+          tenants: 5
+          users_per_tenant: 5
+        quotas:
+          neutron:
+            network: -1
+            subnet: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_update_routers:
+    -
+      args:
+        network_create_args: {}
+        subnet_create_args: {}
+        subnet_cidr_start: "1.1.0.0/30"
+        subnets_per_network: 2
+        router_create_args: {}
+        router_update_args:
+            admin_state_up: False
+            name: "_router_updated"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 5
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+            subnet: -1
+            router: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_update_ports:
+    -
+      args:
+        network_create_args: {}
+        port_create_args: {}
+        ports_per_network: 5
+        port_update_args:
+            admin_state_up: False
+            device_id: "dummy_id"
+            device_owner: "dummy_owner"
+            name: "_port_updated"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 5
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+            port: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_delete_networks:
+    -
+      args:
+        network_create_args: {}
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+            subnet: -1
+      sla:
+        max_failure_percent: 0
+
+  NeutronNetworks.create_and_delete_subnets:
+    -
+      args:
+        network_create_args: {}
+        subnet_create_args: {}
+        subnet_cidr_start: "1.1.0.0/30"
+        subnets_per_network: 2
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+            subnet: -1
+      sla:
+         max_failure_percent: 0
+
+  NeutronNetworks.create_and_delete_ports:
+    -
+      args:
+        network_create_args: {}
+        port_create_args: {}
+        ports_per_network: 10
+      runner:
+        type: "constant"
+        times: 40
+        concurrency: 20
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        quotas:
+          neutron:
+            network: -1
+            port: -1
+      sla:
+          max_failure_percent: 0
+
+  NovaServers.boot_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+      runner:
+        type: "constant"
+        times: 6
+        concurrency: 3
+      context:
+        users:
+          tenants: 3
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.boot_and_delete_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+      runner:
+        type: "constant"
+        times: 6
+        concurrency: 3
+      context:
+        users:
+          tenants: 3
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.boot_and_list_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+        detailed: True
+      runner:
+        type: "constant"
+        times: 6
+        concurrency: 3
+      context:
+        users:
+          tenants: 3
+          users_per_tenant: 1
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.boot_and_bounce_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+        actions:
+          -
+            hard_reboot: 1
+          -
+            soft_reboot: 1
+          -
+            stop_start: 1
+          -
+            rescue_unrescue: 1
+      runner:
+        type: "constant"
+        times: 2
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.boot_server_from_volume_and_delete:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+        volume_size: 1
+      runner:
+        type: "constant"
+        times: 2
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.boot_server_from_volume:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+        volume_size: 1
+      runner:
+        type: "constant"
+        times: 2
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.snapshot_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+      runner:
+        type: "constant"
+        times: 2
+        concurrency: 4
+      context:
+        users:
+          tenants: 3
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  NovaServers.resize_server:
+    -
+      args:
+        flavor:
+            name: "m1.tiny"
+        image:
+            name: "^cirros.*uec$"
+        to_flavor:
+            name: "m1.small"
+        confirm: true
+      runner:
+        type: "constant"
+        times: 2
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
+
+  HeatStacks.create_and_delete_stack:
+    -
+      args:
+        template_path: '/home/jenkins/.rally/extra/server_with_ports.yaml.template'
+      runner:
+        type: "constant"
+        times: 4
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+      sla:
+        max_failure_percent: 0

rally-jobs/rally-zaqar.yaml

@@ -0,0 +1,11 @@
+---
+  ZaqarBasic.create_queue:
+    -
+      args:
+        name_length: 10
+      runner:
+        type: "constant"
+        times: 100
+        concurrency: 10
+      sla:
+        max_failure_percent: 0

tests/ci/rally-gate.sh

@@ -15,9 +15,15 @@
 # This script is executed by post_test_hook function in desvstack gate.
 
 PROJECT=`echo $ZUUL_PROJECT | cut -d \/ -f 2`
-SCENARIO=$BASE/new/$PROJECT/rally-scenarios/${RALLY_SCENARIO}.yaml
-PLUGINS_DIR=$BASE/new/$PROJECT/rally-scenarios/plugins
-EXTRA_DIR=$BASE/new/$PROJECT/rally-scenarios/extra
+
+RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-scenarios
+if [ ! -d $RALLY_JOB_DIR ]; then
+    RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs
+fi
+
+SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml
+PLUGINS_DIR=${RALLY_JOB_DIR}/plugins
+EXTRA_DIR=${RALLY_JOB_DIR}/extra
 
 RALLY_PLUGINS_DIR=~/.rally/plugins
 

tests/unit/rally_jobs/test_jobs.py

@@ -0,0 +1,47 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import traceback
+
+import mock
+import yaml
+
+from rally.benchmark import engine
+import rally.utils as rutils
+from tests.unit import test
+
+
+class RallyJobsTestCase(test.TestCase):
+    rally_jobs_path = os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "rally-jobs")
+
+    @mock.patch("rally.benchmark.engine.BenchmarkEngine"
+                "._validate_config_semantic")
+    def test_schema_is_valid(self, mock_validate):
+        rutils.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
+
+        for filename in ["rally.yaml", "rally-neutron.yaml",
+                         "rally-zaqar.yaml", "rally-designate.yaml"]:
+            full_path = os.path.join(self.rally_jobs_path, filename)
+
+            with open(full_path) as task_file:
+                try:
+                    task_config = yaml.safe_load(task_file.read())
+                    eng = engine.BenchmarkEngine(task_config,
+                                                 mock.MagicMock())
+                    eng.validate()
+                except Exception:
+                    print(traceback.format_exc())
+                    self.fail("Wrong task input file: %s" % full_path)