diff --git a/.testr.conf b/.testr.conf
new file mode 100644
index 0000000..1641f86
--- /dev/null
+++ b/.testr.conf
@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list
diff --git a/doc/source/test_plans/index.rst b/doc/source/test_plans/index.rst
index 6ecce67..74ee21f 100644
--- a/doc/source/test_plans/index.rst
+++ b/doc/source/test_plans/index.rst
@@ -9,5 +9,5 @@ Test Plans
.. toctree::
:maxdepth: 2
- mq/index
- provisioning/main
+ mq/plan
+ provisioning/plan
diff --git a/doc/source/test_plans/mq/index.rst b/doc/source/test_plans/mq/index.rst
deleted file mode 100644
index af37a5c..0000000
--- a/doc/source/test_plans/mq/index.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-Message Queue Test Plan
-=======================
-
-.. toctree::
- :maxdepth: 2
-
- setup
- test_cases
diff --git a/doc/source/test_plans/mq/setup.rst b/doc/source/test_plans/mq/plan.rst
similarity index 52%
rename from doc/source/test_plans/mq/setup.rst
rename to doc/source/test_plans/mq/plan.rst
index bc55efa..24eeeb5 100644
--- a/doc/source/test_plans/mq/setup.rst
+++ b/doc/source/test_plans/mq/plan.rst
@@ -1,5 +1,20 @@
-Test Setup
-----------
+=======================
+Message Queue Test Plan
+=======================
+
+:status: ready
+:version: 0
+
+:Abstract:
+
+ This document describes a test plan for quantifying the performance of
+ message queues usually used as a message bug between OpenStack services.
+
+Test Plan
+=========
+
+Test Environment
+----------------
This section describes the setup for message queue testing. It can be either
a single (all-in-one) or a multi-node installation.
@@ -15,14 +30,17 @@ A basic multi-node setup with RabbitMQ or ActiveMQ comprises 5 physical nodes:
is typical for OpenStack control plane services.
* Three nodes are allocated for the MQ cluster.
-When using ZeroMQ, the basic multi-node setup can be reduced to two physical nodes.
+When using ZeroMQ, the basic multi-node setup can be reduced to two physical
+nodes.
+
* One node for a compute node as above.
- * One node for a controller node. This node also acts as a Redis host
- for match making purposes.
+ * One node for a controller node. This node also acts as a Redis host for
+ match making purposes.
+Preparation
+^^^^^^^^^^^
-RabbitMQ Installation and Configuration
----------------------------------------
+**RabbitMQ Installation and Configuration**
* Install RabbitMQ server package:
``sudo apt-get install rabbitmq-server``
@@ -51,8 +69,7 @@ RabbitMQ Installation and Configuration
``sudo rabbitmqctl set_permissions stackrabbit ".*" ".*" ".*"``
-ActiveMQ Installation and Configuration
----------------------------------------
+**ActiveMQ Installation and Configuration**
This section describes installation and configuration steps for an ActiveMQ
message queue implementation. ActiveMQ is based on Java technologies so it
@@ -79,8 +96,8 @@ performed for an ActiveMQ installation:
.. note::
Here 10.4.1.x are the IP addresses of the ZooKeeper nodes where ZK is
- installed. ZK will be run in cluster mode with majority voting, so at least 3 nodes
- are required.
+ installed. ZK will be run in cluster mode with majority voting, so at least
+ 3 nodes are required.
.. code-block:: none
@@ -96,8 +113,8 @@ performed for an ActiveMQ installation:
* create dataDir and dataLogDir directories
* for each MQ node create a myid file in dataDir with the id of the
- server and nothing else. For node-1 the file will contain one line with 1,
- node-2 with 2, and node-3 with 3.
+ server and nothing else. For node-1 the file will contain one line
+ with 1, node-2 with 2, and node-3 with 3.
* start ZooKeeper (on each node): \textbf{./zkServer.sh start}
* check ZK status with: \textbf{./zkServer.sh status}
* Configure ActiveMQ (apache-activemq-5.12.0/conf/activemq.xml file - set
@@ -134,27 +151,28 @@ After ActiveMQ is installed and configured it can be started with the command:
:command:./activemq start or ``./activemq console`` for a foreground process.
-Oslo.messaging ActiveMQ Driver
-------------------------------
+**Oslo.messaging ActiveMQ Driver**
+
+
+All OpenStack changes (in the oslo.messaging library) to support ActiveMQ are
+already merged to the upstream repository. The relevant changes can be found in
+the amqp10-driver-implementation topic.
-All OpenStack changes (in the oslo.messaging library) to support ActiveMQ are already
-merged to the upstream repository. The relevant changes can be found in the
-amqp10-driver-implementation topic.
To run ActiveMQ even on the most basic all-in-one topology deployment the
following requirements need to be satisfied:
- * Java JRE must be installed in the system. The Java version can be checked with the
- command ``java -version``. If java is not installed an error message will
- appear. Java can be installed with the following command:
+ * Java JRE must be installed in the system. The Java version can be checked
+ with the command ``java -version``. If java is not installed an error
+ message will appear. Java can be installed with the following command:
``sudo apt-get install default-jre``
* ActiveMQ binaries should be installed in the system. See
- http://activemq.apache.org/getting-started.html for installation instructions.
- The latest stable version is currently
+ http://activemq.apache.org/getting-started.html for installation
+ instructions. The latest stable version is currently
http://apache-mirror.rbc.ru/pub/apache/activemq/5.12.0/apache-activemq-5.12.0-bin.tar.gz.
- * To use the OpenStack oslo.messaging amqp 1.0 driver, the following Python libraries
- need to be installed:
+ * To use the OpenStack oslo.messaging amqp 1.0 driver, the following Python
+ libraries need to be installed:
``pip install "pyngus$>=$1.0.0,$<$2.0.0"``
``pip install python-qpid-proton``
@@ -162,19 +180,20 @@ following requirements need to be satisfied:
``rpc_backend = rabbit`` need to be modified to replace this line with
``rpc_backend = amqp``, and then all the services need to be restarted.
-ZeroMQ Installation
--------------------
+**ZeroMQ Installation**
This section describes installation steps for ZeroMQ. ZeroMQ (also ZMQ or 0MQ)
is an embeddable networking library but acts like a concurrency framework.
-Unlike other AMQP-based drivers, such as RabbitMQ, ZeroMQ doesn’t have any central brokers in
-oslo.messaging. Instead, each host (running OpenStack services) is both a ZeroMQ client and
-a server. As a result, each host needs to listen to a certain TCP port for incoming connections
-and directly connect to other hosts simultaneously.
+Unlike other AMQP-based drivers, such as RabbitMQ, ZeroMQ doesn’t have any
+central brokers in oslo.messaging. Instead, each host (running OpenStack
+services) is both a ZeroMQ client and a server. As a result, each host needs to
+listen to a certain TCP port for incoming connections and directly connect to
+other hosts simultaneously.
To set up ZeroMQ, only one step needs to be performed.
- * Install python bindings for ZeroMQ. All necessary packages will be installed as dependencies:
+ * Install python bindings for ZeroMQ. All necessary packages will be
+ installed as dependencies:
``sudo apt-get install python-zmq``
.. note::
@@ -191,11 +210,12 @@ To set up ZeroMQ, only one step needs to be performed.
Depends: libc6
Depends: libzmq3
-Oslo.messaging ZeroMQ Driver
-----------------------------
-All OpenStack changes (in the oslo.messaging library) to support ZeroMQ are already
-merged to the upstream repository. You can find the relevant changes in the
-zmq-patterns-usage topic.
+**Oslo.messaging ZeroMQ Driver**
+
+All OpenStack changes (in the oslo.messaging library) to support ZeroMQ are
+already merged to the upstream repository. You can find the relevant changes
+in the zmq-patterns-usage topic.
+
To run ZeroMQ on the most basic all-in-one topology deployment the
following requirements need to be satisfied:
@@ -206,11 +226,12 @@ following requirements need to be satisfied:
.. note::
- The following changes need to be applied to all OpenStack project configuration files.
+ The following changes need to be applied to all OpenStack project
+ configuration files.
- * To enable the driver, in the section [DEFAULT] of each configuration file, the ‘rpc_backend’
- flag must be set to ‘zmq’ and the ‘rpc_zmq_host’ flag must be set to the hostname
- of the node.
+ * To enable the driver, in the section [DEFAULT] of each configuration file,
+ the ‘rpc_backend’ flag must be set to ‘zmq’ and the ‘rpc_zmq_host’ flag
+ must be set to the hostname of the node.
.. code-block:: none
@@ -231,19 +252,20 @@ following requirements need to be satisfied:
port = 6379
password = None
-Running ZeroMQ on a multi-node setup
-------------------------------------
-The process of setting up oslo.messaging with ZeroMQ on a multi-node environment is very similar
-to the all-in-one installation.
+**Running ZeroMQ on a multi-node setup**
+
+The process of setting up oslo.messaging with ZeroMQ on a multi-node
+environment is very similar to the all-in-one installation.
* On each node ``rpc_zmq_host`` should be set to its FQDN.
- * Redis-server should be up and running on a controller node or a separate host.
- Redis can be used with master-slave replication enabled, but currently the oslo.messaging ZeroMQ driver
- does not support Redis Sentinel, so it is not yet possible to achieve high availability, automatic failover,
+ * Redis-server should be up and running on a controller node or a separate
+ host. Redis can be used with master-slave replication enabled, but
+ currently the oslo.messaging ZeroMQ driver does not support Redis Sentinel,
+ so it is not yet possible to achieve high availability, automatic failover,
and fault tolerance.
- The ``host`` parameter in section ``[matchmaker_redis]`` should be set to the IP address of a host which runs
- a master Redis instance, e.g.
+ The ``host`` parameter in section ``[matchmaker_redis]`` should be set to
+ the IP address of a host which runs a master Redis instance, e.g.
.. code-block:: none
@@ -251,3 +273,119 @@ to the all-in-one installation.
host = 10.0.0.3
port = 6379
password = None
+
+Environment description
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Test results must include used environment description. This includes:
+
+* Hardware used (servers, switches, storage, etc.)
+* Network scheme
+* Messaging bus specification and OpenStack version deployed (if any).
+
+
+Test Case 1: Message Queue Throughput Test
+------------------------------------------
+
+Description
+^^^^^^^^^^^
+
+This test measures the aggregate throughput of a MQ layer by using the
+oslo.messaging simulator tool. Either RabbitMQ, ActiveMQ, or ZeroMQ can be used
+as the MQ layer. Throughput is calculated as the sum over the MQ clients of the
+throughput for each client. For each test the number of clients/threads is
+configured to one of the specific values defined in the test case parameters
+section. The full set of tests will cover all the "Threads count" values shown,
+plus additional values as needed to quantify the dependence of MQ throughput on
+load, and to find the maximum throughput.
+
+Parameters
+^^^^^^^^^^
+
+======================= ===========
+Parameter name Value
+======================= ===========
+oslo.messaging version 2.5.0
+simulator.py version 1.0
+Threads count 50, 70, 100
+======================= ===========
+
+List of performance metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+======== ========== ================ ===================================
+Priority Value Measurment Units Description
+======== ========== ================ ===================================
+1 Throughput msg/sec Directly measured by simulator tool
+======== ========== ================ ===================================
+
+Result Type
+^^^^^^^^^^^
+
+================ ======================= =========================
+Result type Measurement Units Description
+================ ======================= =========================
+Throughput Value msg/sec Table of numerical values
+Throughput Graph msg/sec vs # of threads Graph
+================ ======================= =========================
+
+Additional Measurements
+^^^^^^^^^^^^^^^^^^^^^^^
+
+=========== ======= =============================
+Measurement Units Description
+=========== ======= =============================
+Variance msg/sec Throughput variance over time
+=========== ======= =============================
+
+Test Case 2: OMGBenchmark Rally test
+------------------------------------
+
+Description
+^^^^^^^^^^^
+
+OMGBenchmark is a rally plugin for benchmarking oslo.messaging.
+The plugin and installation instructions are available on github:
+https://github.com/Yulya/omgbenchmark
+
+Parameters
+^^^^^^^^^^
+
+================================= =============== ===============
+Parameter name Rally name Value
+================================= =============== ===============
+oslo.messaging version 2.5.0
+Number of iterations times 50, 100, 500
+Threads count concurrency 40, 70, 100
+Number of RPC servers num_servers 10
+Number of RPC clients num_clients 10
+Number of topics num_topics 5
+Number of messages per iteration num_messages 100
+Message size msg_length_file 900-12000 bytes
+================================= =============== ===============
+
+List of performance metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+======= ================= ==========================================
+Name Measurement Units Description
+======= ================= ==========================================
+min sec Minimal execution time of one iteration
+median sec Median execution time
+90%ile sec 90th percentile execution time
+95%ile sec 95th percentile execution time
+max sec Maximal execution time of one iteration
+avg sec Average execution time
+success none Number of successfully finished iterations
+count none Number of executed iterations
+======= ================= ==========================================
+
+Result Type
+^^^^^^^^^^^
+
+================= ======================= =========================
+Result type Measurement Units Description
+================= ======================= =========================
+Throughput Graph msg size vs median Graph
+Concurrency Graph concurrency vs median Graph
+================= ======================= =========================
diff --git a/doc/source/test_plans/mq/test_cases.rst b/doc/source/test_plans/mq/test_cases.rst
deleted file mode 100644
index 2844f1f..0000000
--- a/doc/source/test_plans/mq/test_cases.rst
+++ /dev/null
@@ -1,99 +0,0 @@
-Test Cases
-==========
-
-Test Case 1: Message Queue Throughput Test
-------------------------------------------
-
-**Description**
-
-This test measures the aggregate throughput of a MQ layer by using the oslo.messaging
-simulator tool. Either RabbitMQ, ActiveMQ, or ZeroMQ can be used as the MQ layer.
-Throughput is calculated as the sum
-over the MQ clients of the throughput for each client. For each test the number of
-clients/threads is configured to one of the specific values defined in the test case
-parameters section. The full set of tests will cover all the "Threads count" values shown,
-plus additional values as needed to quantify the dependence of MQ throughput on load, and
-to find the maximum throughput.
-
-**Parameters**
-
-======================= =====
-Parameter name Value
-======================= =====
-oslo.messaging version 2.5.0
-simulator.py version 1.0
-Threads count 50, 70, 100
-======================= =====
-
-**Measurements**
-
-========== ================ ===========
-Value Measurment Units Description
-========== ================ ===========
-Throughput msg/sec Directly measured by simulator tool
-========== ================ ===========
-
-**Result Type**
-
-================ ======================= =========================
-Result type Measurement Units Description
-================ ======================= =========================
-Throughput Value msg/sec Table of numerical values
-Throughput Graph msg/sec vs # of threads Graph
-================ ======================= =========================
-
-**Additional Measurements**
-
-=========== ======= =============================
-Measurement Units Description
-=========== ======= =============================
-Variance msg/sec Throughput variance over time
-=========== ======= =============================
-
-Test Case 2: OMGBenchmark Rally test
-------------------------------------
-
-**Description**
-
-OMGBenchmark is a rally plugin for benchmarking oslo.messaging.
-The plugin and installation instructions are available on github:
-https://github.com/Yulya/omgbenchmark
-
-**Parameters**
-
-================================= =============== =====
-Parameter name Rally name Value
-================================= =============== =====
-oslo.messaging version 2.5.0
-Number of iterations times 50, 100, 500
-Threads count concurrency 40, 70, 100
-Number of RPC servers num_servers 10
-Number of RPC clients num_clients 10
-Number of topics num_topics 5
-Number of messages per iteration num_messages 100
-Message size msg_length_file 900-12000 bytes
-================================= =============== =====
-
-**Measurements**
-
-======= ================= ==========================================
-Name Measurement Units Description
-======= ================= ==========================================
-min sec Minimal execution time of one iteration
-median sec Median execution time
-90%ile sec 90th percentile execution time
-95%ile sec 95th percentile execution time
-max sec Maximal execution time of one iteration
-avg sec Average execution time
-success none Number of successfully finished iterations
-count none Number of executed iterations
-======= ================= ==========================================
-
-**Result Type**
-
-================= ======================= =========================
-Result type Measurement Units Description
-================= ======================= =========================
-Throughput Graph msg size vs median Graph
-Concurrency Graph concurrency vs median Graph
-================= ======================= =========================
diff --git a/doc/source/test_plans/provisioning/main.rst b/doc/source/test_plans/provisioning/plan.rst
similarity index 66%
rename from doc/source/test_plans/provisioning/main.rst
rename to doc/source/test_plans/provisioning/plan.rst
index 957db60..67d8306 100644
--- a/doc/source/test_plans/provisioning/main.rst
+++ b/doc/source/test_plans/provisioning/plan.rst
@@ -10,20 +10,20 @@ Measuring performance of provisioning systems
:Abstract:
This document describes a test plan for quantifying the performance of
- provisioning systems as a function of the number of nodes to be provisioned. The
- plan includes the collection of several resource utilization metrics, which will
- be used to analyze and understand the overall performance of each system. In
- particular, resource bottlenecks will either be fixed, or best practices
- developed for system configuration and hardware requirements.
+ provisioning systems as a function of the number of nodes to be provisioned.
+ The plan includes the collection of several resource utilization metrics,
+ which will be used to analyze and understand the overall performance of each
+ system. In particular, resource bottlenecks will either be fixed, or best
+ practices developed for system configuration and hardware requirements.
:Conventions:
- **Provisioning:** is the entire process of installing and configuring an
operating system.
- - **Provisioning system:** is a service or a set of services which enables the
- installation of an operating system and performs basic operations such as
- configuring network interfaces and partitioning disks. A preliminary
+ - **Provisioning system:** is a service or a set of services which enables
+ the installation of an operating system and performs basic operations such
+ as configuring network interfaces and partitioning disks. A preliminary
`list of provisioning systems`_ can be found below in `Applications`_.
The provisioning system
can include configuration management systems like Puppet or Chef, but
@@ -37,70 +37,38 @@ Measuring performance of provisioning systems
- **Nodes:** are servers which will be provisioned.
-List of performance metrics
----------------------------
-The table below shows the list of test metrics to be collected. The priority
-is the relative ranking of the importance of each metric in evaluating the
-performance of the system.
-
-.. table:: List of performance metrics
-
- +--------+------------------------+------------------------------------------+
- |Priority| Parameter | Description |
- +========+========================+==========================================+
- | | | | The elapsed time to provision all |
- | 1 |PROVISIONING_TIME(NODES)| | nodes, as a function of the numbers of |
- | | | | nodes |
- +--------+------------------------+------------------------------------------+
- | | | | Incoming network bandwidth usage as a |
- | 2 |INGRESS_NET(NODES) | | function of the number of nodes. |
- | | | | Average during provisioning on the host|
- | | | | where the provisioning system is |
- | | | | installed. |
- +--------+------------------------+------------------------------------------+
- | | | | Outgoing network bandwidth usage as a |
- | 2 | EGRESS_NET(NODES) | | function of the number of nodes. |
- | | | | Average during provisioning on the host|
- | | | | where the provisioning system is |
- | | | | installed. |
- +--------+------------------------+------------------------------------------+
- | | | | CPU utilization as a function of the |
- | 3 | CPU(NODES) | | number of nodes. Average during |
- | | | | provisioning on the host where the |
- | | | | provisioning system is installed. |
- +--------+------------------------+------------------------------------------+
- | | | | Active memory usage as a function of |
- | 3 | RAM(NODES) | | the number of nodes. Average during |
- | | | | provisioning on the host where the |
- | | | | provisioning system is installed. |
- +--------+------------------------+------------------------------------------+
- | | | | Storage read IO bandwidth as a |
- | 3 | WRITE_IO(NODES) | | function of the number of nodes. |
- | | | | Average during provisioning on the host|
- | | | | where the provisioning system is |
- | | | | installed. |
- +--------+------------------------+------------------------------------------+
- | | | | Storage write IO bandwidth as a |
- | 3 | READ_IO(NODES) | | function of the number of nodes. |
- | | | | Average during provisioning on the host|
- | | | | where the provisioning system is |
- | | | | installed. |
- +--------+------------------------+------------------------------------------+
-
Test Plan
----------
+=========
-The above performance metrics will be measured for various number
-of provisioned nodes. The result will be a table that shows the
-dependence of these metrics on the number of nodes.
+This test plan aims to identify the best provisioning solution for cloud
+deployment, using specified list of performance measurements and tools.
+
+Test Environment
+----------------
+
+Preparation
+^^^^^^^^^^^
+
+1.
+ The following package needs to be installed on the provisioning system
+ servers to collect performance metrics.
+
+.. table:: Software to be installed
+
+ +--------------+---------+-----------------------------------+
+ | package name | version | source |
+ +==============+=========+===================================+
+ | `dstat`_ | 0.7.2 | Ubuntu trusty universe repository |
+ +--------------+---------+-----------------------------------+
Environment description
^^^^^^^^^^^^^^^^^^^^^^^
-Test results MUST include a description of the environment used. The following items
-should be included:
-- **Hardware configuration of each server.** If virtual machines are used then both
- physical and virtual hardware should be fully documented.
+Test results MUST include a description of the environment used. The following
+items should be included:
+
+- **Hardware configuration of each server.** If virtual machines are used then
+ both physical and virtual hardware should be fully documented.
An example format is given below:
.. table:: Description of server hardware
@@ -141,10 +109,10 @@ should be included:
| |size | | |
+-------+----------------+-------+-------+
-- **Configuration of hardware network switches.** The configuration file from the
- switch can be downloaded and attached.
+- **Configuration of hardware network switches.** The configuration file from
+ the switch can be downloaded and attached.
-- **Configuration of virtual machines and virtual networks (if they are used).**
+- **Configuration of virtual machines and virtual networks (if used).**
The configuration files can be attached, along with the mapping of virtual
machines to host machines.
@@ -166,22 +134,78 @@ should be included:
affect the amount of work to be performed by the provisioning system
and thus its performance.
-Preparation
+Test Case
+---------
+
+Description
^^^^^^^^^^^
-1.
- The following package needs to be installed on the provisioning system
- servers to collect performance metrics.
-.. table:: Software to be installed
+This specific test plan contains only one test case, that needs to be run
+step by step on the environments differing list of parameters below.
- +--------------+---------+-----------------------------------+
- | package name | version | source |
- +==============+=========+===================================+
- | `dstat`_ | 0.7.2 | Ubuntu trusty universe repository |
- +--------------+---------+-----------------------------------+
+Parameters
+^^^^^^^^^^
+
+=============== =========================================
+Parameter name Value
+=============== =========================================
+number of nodes 10, 20, 40, 80, 160, 320, 640, 1280, 2000
+=============== =========================================
+
+List of performance metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The table below shows the list of test metrics to be collected. The priority
+is the relative ranking of the importance of each metric in evaluating the
+performance of the system.
+
+.. table:: List of performance metrics
+
+ +--------+-------------------+-------------------+------------------------------------------+
+ |Priority| Value | Measurement Units | Description |
+ +========+===================+===================+==========================================+
+ | | | || The elapsed time to provision all |
+ | 1 | PROVISIONING_TIME | seconds || nodes, as a function of the numbers of |
+ | | | || nodes |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || Incoming network bandwidth usage as a |
+ | 2 | INGRESS_NET | Gbit/s || function of the number of nodes. |
+ | | | || Average during provisioning on the host |
+ | | | || where the provisioning system is |
+ | | | || installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || Outgoing network bandwidth usage as a |
+ | 2 | EGRESS_NET | Gbit/s || function of the number of nodes. |
+ | | | || Average during provisioning on the host |
+ | | | || where the provisioning system is |
+ | | | || installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || CPU utilization as a function of the |
+ | 3 | CPU | percentage || number of nodes. Average during |
+ | | | || provisioning on the host where the |
+ | | | || provisioning system is installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || Active memory usage as a function of |
+ | 3 | RAM | GB || the number of nodes. Average during |
+ | | | || provisioning on the host where the |
+ | | | || provisioning system is installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || Storage read IO bandwidth as a |
+ | 3 | WRITE_IO | operations/second || function of the number of nodes. |
+ | | | || Average during provisioning on the host |
+ | | | || where the provisioning system is |
+ | | | || installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
+ | | | || Storage write IO bandwidth as a |
+ | 3 | READ_IO | operations/second || function of the number of nodes. |
+ | | | || Average during provisioning on the host |
+ | | | || where the provisioning system is |
+ | | | || installed. |
+ +--------+-------------------+-------------------+------------------------------------------+
Measuring performance values
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
The script
`Full script for collecting performance metrics`_
can be used for the first five of the following steps.
@@ -197,9 +221,8 @@ can be used for the first five of the following steps.
2.
Start the provisioning process for the first node and record the wall time.
3.
- Wait until the provisioning process has finished (when all nodes are reachable
- via ssh)
- and record the wall time.
+ Wait until the provisioning process has finished (when all nodes are
+ reachable via ssh) and record the wall time.
4.
Stop the dstat program.
5.
@@ -233,32 +256,20 @@ can be used for the first five of the following steps.
These values will be graphed and maximum values reported.
-6.
- Repeat steps 1-5 for provisioning at the same time the following number of
- nodes:
-
- * 10 nodes
- * 20 nodes
- * 40 nodes
- * 80 nodes
- * 160 nodes
- * 320 nodes
- * 640 nodes
- * 1280 nodes
- * 2000 nodes
-
Additional tests will be performed if some anomalous behaviour is found.
These may require the collection of additional performance metrics.
-7.
+6.
The result of this part of test will be:
* to provide the following graphs, one for each number of provisioned nodes:
#) Three dependencies on one graph.
- * INGRESS_NET(TIME) Dependence on time of incoming network bandwidth usage.
- * EGRESS_NET(TIME) Dependence on time of outgoing network bandwidth usage.
+ * INGRESS_NET(TIME) Dependence on time of incoming network bandwidth
+ usage.
+ * EGRESS_NET(TIME) Dependence on time of outgoing network bandwidth
+ usage.
* ALL_NET(TIME) Dependence on time of total network bandwidth usage.
#) One dependence on one graph.
@@ -313,10 +324,10 @@ nodes.
+-------+--------------+---------+---------+---------+---------+
Applications
-------------
+============
-list of provisioning systems
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+List of provisioning systems
+----------------------------
.. table:: list of provisioning systems
@@ -333,7 +344,7 @@ list of provisioning systems
+-----------------------------+---------+
Full script for collecting performance metrics
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+==============================================
.. literalinclude:: measure.sh
:language: bash
diff --git a/doc/source/test_plans/template.rst b/doc/source/test_plans/template.rst
index fbe237c..864ccc0 100644
--- a/doc/source/test_plans/template.rst
+++ b/doc/source/test_plans/template.rst
@@ -2,8 +2,6 @@
Example Test Plan - The title of your plan
==========================================
-Please include the following information to this primary section:
-
:status: test plan status - either **draft** or **ready**
:version: test plan version
@@ -33,13 +31,15 @@ using sections, similar to the written below.
Test Environment
----------------
-**Preparation**
+Preparation
+^^^^^^^^^^^
Please specify here what needs to be done with the environment to run
this test plan. This can include specific tools installation,
specific OpenStack deployment, etc.
-**Environment description**
+Environment description
+^^^^^^^^^^^^^^^^^^^^^^^
Please define here used environment. You can use the scheme below for this
purpose or modify it due to your needs:
@@ -54,17 +54,20 @@ purpose or modify it due to your needs:
Test Case 1: Something very interesting #1
------------------------------------------
-**Description**
+Description
+^^^^^^^^^^^
Define test case #1. Every test case can contain at least the sections, defined
below.
-**Parameters**
+Parameters
+^^^^^^^^^^
Optional section. Can be used if there are multiple test cases differing in
some input parameters - if so, these parameters need to be listed here.
-**List of performance metrics**
+List of performance metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
Mandatory section. Defines what measurements are in fact done during the test.
To be a good citizen in case of multiple metrics collection, it will be nice to
@@ -78,7 +81,8 @@ Priority Value Measurement Units Description
3 - not that much important What's measured <units> <description>
=========================== =============== ================= =============
-**Some additional section**
+Some additional section
+^^^^^^^^^^^^^^^^^^^^^^^
Depending on the test case nature, something else may need to be defined.
If so, additional sections with free form titles should be added.
diff --git a/requirements.txt b/requirements.txt
index 2d4bd5e..1dde9cc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,5 @@ rst2pdf
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
sphinxcontrib-httpdomain
sphinx_rtd_theme
+testrepository>=0.0.18
+testtools>=0.9.34
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_titles.py b/tests/test_titles.py
new file mode 100644
index 0000000..811cd73
--- /dev/null
+++ b/tests/test_titles.py
@@ -0,0 +1,198 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import glob
+import re
+
+import docutils.core
+import testtools
+
+
+OPTIONAL_SECTIONS = ("Upper level additional section",)
+OPTIONAL_SUBSECTIONS = ("Some additional section",)
+OPTIONAL_SUBSUBSECTIONS = ("Parameters", "Some additional section",)
+OPTIONAL_FIELDS = ("Conventions",)
+
+
+class TestTitles(testtools.TestCase):
+ def _get_title(self, section_tree, depth=1):
+ section = {
+ "subtitles": [],
+ }
+ for node in section_tree:
+ if node.tagname == "title":
+ section["name"] = node.rawsource
+ elif node.tagname == "section":
+ subsection = self._get_title(node, depth+1)
+ if depth < 2:
+ if subsection["subtitles"]:
+ section["subtitles"].append(subsection)
+ else:
+ section["subtitles"].append(subsection["name"])
+ elif depth == 2:
+ section["subtitles"].append(subsection["name"])
+ return section
+
+ def _get_titles(self, test_plan):
+ titles = {}
+ for node in test_plan:
+ if node.tagname == "section":
+ section = self._get_title(node)
+ titles[section["name"]] = section["subtitles"]
+ return titles
+
+ @staticmethod
+ def _get_docinfo(test_plan):
+ fields = []
+ for node in test_plan:
+ if node.tagname == "field_list":
+ for field in node:
+ for f_opt in field:
+ if f_opt.tagname == "field_name":
+ fields.append(f_opt.rawsource)
+
+ if node.tagname == "docinfo":
+ for info in node:
+ fields.append(info.tagname)
+
+ if node.tagname == "topic":
+ fields.append("abstract")
+
+ return fields
+
+ def _check_fields(self, tmpl, test_plan):
+ tmpl_fields = self._get_docinfo(tmpl)
+ test_plan_fields = self._get_docinfo(test_plan)
+
+ missing_fields = [f for f in tmpl_fields
+ if f not in test_plan_fields and
+ f not in OPTIONAL_FIELDS]
+
+ if len(missing_fields) > 0:
+ self.fail("While checking '%s':\n %s"
+ % (test_plan[0].rawsource,
+ "Missing fields: %s" % missing_fields))
+
+ def _check_titles(self, filename, expect, actual):
+ missing_sections = [x for x in expect.keys() if (
+ x not in actual.keys()) and (x not in OPTIONAL_SECTIONS)]
+
+ msgs = []
+ if len(missing_sections) > 0:
+ msgs.append("Missing sections: %s" % missing_sections)
+
+ for section in expect.keys():
+ missing_subsections = [x for x in expect[section]
+ if x not in actual.get(section, {}) and
+ (x not in OPTIONAL_SUBSECTIONS)]
+ extra_subsections = [x for x in actual.get(section, {})
+ if x not in expect[section]]
+
+ for ex_s in extra_subsections:
+ s_name = (ex_s if type(ex_s) is str or
+ type(ex_s) is unicode else ex_s["name"])
+ if s_name.startswith("Test Case"):
+ new_missing_subsections = []
+ for m_s in missing_subsections:
+ m_s_name = (m_s if type(m_s) is str or
+ type(m_s) is unicode
+ else m_s["name"])
+ if not m_s_name.startswith("Test Case"):
+ new_missing_subsections.append(m_s)
+ missing_subsections = new_missing_subsections
+ break
+
+ if len(missing_subsections) > 0:
+ msgs.append("Section '%s' is missing subsections: %s"
+ % (section, missing_subsections))
+
+ for subsection in expect[section]:
+ if type(subsection) is dict:
+ missing_subsubsections = []
+ actual_section = actual.get(section, {})
+ matching_actual_subsections = [
+ s for s in actual_section
+ if type(s) is dict and (
+ s["name"] == subsection["name"] or
+ (s["name"].startswith("Test Case") and
+ subsection["name"].startswith("Test Case")))
+ ]
+ for actual_subsection in matching_actual_subsections:
+ for x in subsection["subtitles"]:
+ if (x not in actual_subsection["subtitles"] and
+ x not in OPTIONAL_SUBSUBSECTIONS):
+ missing_subsubsections.append(x)
+ if len(missing_subsubsections) > 0:
+ msgs.append("Subsection '%s' is missing "
+ "subsubsections: %s"
+ % (actual_subsection,
+ missing_subsubsections))
+
+ if len(msgs) > 0:
+ self.fail("While checking '%s':\n %s"
+ % (filename, "\n ".join(msgs)))
+
+ def _check_lines_wrapping(self, tpl, raw):
+ code_block = False
+ for i, line in enumerate(raw.split("\n")):
+ # NOTE(ndipanov): Allow code block lines to be longer than 79 ch
+ if code_block:
+ if not line or line.startswith(" "):
+ continue
+ else:
+ code_block = False
+ if "::" in line:
+ code_block = True
+ if "http://" in line or "https://" in line:
+ continue
+ # Allow lines which do not contain any whitespace
+ if re.match("\s*[^\s]+$", line):
+ continue
+ self.assertTrue(
+ len(line) < 80,
+ msg="%s:%d: Line limited to a maximum of 79 characters." %
+ (tpl, i + 1))
+
+ def _check_no_cr(self, tpl, raw):
+ matches = re.findall("\r", raw)
+ self.assertEqual(
+ len(matches), 0,
+ "Found %s literal carriage returns in file %s" %
+ (len(matches), tpl))
+
+ def _check_trailing_spaces(self, tpl, raw):
+ for i, line in enumerate(raw.split("\n")):
+ trailing_spaces = re.findall("\s+$", line)
+ self.assertEqual(
+ len(trailing_spaces), 0,
+ "Found trailing spaces on line %s of %s" % (i + 1, tpl))
+
+ def test_template(self):
+ with open("doc/source/test_plans/template.rst") as f:
+ template = f.read()
+ test_plan_tmpl = docutils.core.publish_doctree(template)
+ template_titles = self._get_titles(test_plan_tmpl)
+
+ files = glob.glob("doc/source/test_plans/*/*.rst")
+
+ for filename in files:
+ with open(filename) as f:
+ data = f.read()
+
+ test_plan = docutils.core.publish_doctree(data)
+ self._check_titles(filename,
+ template_titles,
+ self._get_titles(test_plan))
+ self._check_fields(test_plan_tmpl, test_plan)
+ self._check_lines_wrapping(filename, data)
+ self._check_no_cr(filename, data)
+ self._check_trailing_spaces(filename, data)
diff --git a/tox.ini b/tox.ini
index d7fcf86..038e393 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = docs
+envlist = docs,py27
minversion = 1.6
skipsdist = True
@@ -11,6 +11,8 @@ setenv = VIRTUAL_ENV={envdir}
LANGUAGE=en_US:en
LC_ALL=C
deps = -r{toxinidir}/requirements.txt
+commands =
+ python setup.py test --slowest --testr-args='{posargs}'
[testenv:venv]
commands = {posargs}