openstack
/
rally
Code Issues Proposed changes

Restruct rally.plugins.common module 

Change-Id: I9df675afdf0a6b76916fba7ed9c3712136ad4780
This commit is contained in:
Andrey Kurilin 2020-03-21 22:51:06 +02:00
parent 208b2b6351
commit 4a466f2bc5
106 changed files with 3333 additions and 2715 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
CHANGELOG.rst
View File

@ -37,7 +37,7 @@ Changed
* *path_or_url* plugin follows redirects while validating urls now.
* *rally task sla-check` fails if there is no data.
* *rally task sla-check* fails if there is no data.
Deprecated
~~~~~~~~~~
@ -45,6 +45,30 @@ Deprecated
* Module *rally.common.sshutils* is deprecated. Use *rally.utils.sshutils*
instead.
* All modules from *rally.plugins.common.contexts* are deprecated. Use
*rally.plugins.task.contexts* instead.
* All modules from *rally.plugins.common.exporters* are deprecated. Use
*rally.plugins.task.exporters* instead.
* Module *rally.plugins.common.hook.sys_call* is deprecated. Use
*rally.plugins.task.hooks.sys_call* instead.
* All modules from *rally.plugins.common.hook.triggers* are deprecated. Use
*rally.plugins.task.hook_triggers* instead.
* All modules from *rally.plugins.common.runners* are deprecated. Use
*rally.plugins.task.runners* instead.
* All modules from *rally.plugins.common.scenarios* are deprecated. Use
*rally.plugins.task.scenarios* instead.
* All modules from *rally.plugins.common.sla* are deprecated. Use
*rally.plugins.task.sla* instead.
* All modules from *rally.plugins.common.verification* are deprecated. Use
*rally.plugins.verification* instead.
Removed
~~~~~~~

9
rally/common/logging.py
View File

@ -15,6 +15,7 @@
import functools
import traceback
import warnings
from oslo_log import handlers
from oslo_log import log as oslogging
@ -331,5 +332,13 @@ def log_deprecated_args(message, rally_version, deprecated_args,
return decorator
def log_deprecated_module(target, new_module, release):
warnings.warn(
f"Module `{target}` moved to `{new_module}` since Rally v{release}. "
f"The import from old place is deprecated and may be removed in "
f"further releases."
)
def is_debug():
return CONF.debug or CONF.rally_debug

10
rally/common/sshutils.py
View File

@ -13,11 +13,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.utils.sshutils import * # noqa
from rally.utils.sshutils import * # noqa: F401,F403
from rally.utils import sshutils as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
logging.getLogger(__name__).warning(
f"Module {__name__} moved to rally.utils.sshutils. "
f"Please correct your import."
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

10
rally/plugins/__init__.py
View File

@ -31,7 +31,15 @@ def load():
opts.register()
discover.import_modules_from_package("rally.plugins.common")
# NOTE(andreykurilin): `rally.plugins.common` includes deprecated
# modules. As soon as they will be removed the direct import of
# validators should be replaced by
#
# discover.import_modules_from_package("rally.plugins.common")
from rally.plugins.common import validators # noqa: F401
discover.import_modules_from_package("rally.plugins.task")
discover.import_modules_from_package("rally.plugins.verification")
packages = discover.find_packages_by_entry_point()
for package in packages:

147
rally/plugins/common/exporters/elastic/client.py
View File

@ -12,148 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import requests
from rally.plugins.task.exporters.elastic.client import * # noqa: F401,F403
from rally.plugins.task.exporters.elastic import client as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
from rally import exceptions
LOG = logging.getLogger(__name__)
class ElasticSearchClient(object):
"""The helper class for communication with ElasticSearch 2.*, 5.*, 6.*"""
# a number of documents to push to the cluster at once.
CHUNK_LENGTH = 10000
def __init__(self, url):
self._url = url.rstrip("/") if url else "http://localhost:9200"
self._version = None
@staticmethod
def _check_response(resp, action=None):
if resp.status_code in (200, 201):
return
# it is an error. let's try to find the reason
reason = None
try:
data = resp.json()
except ValueError:
# it is ok
pass
else:
if "error" in data:
if isinstance(data["error"], dict):
reason = data["error"].get("reason", "")
else:
reason = data["error"]
reason = reason or resp.text or "n/a"
action = action or "connect to"
raise exceptions.RallyException(
"[HTTP %s] Failed to %s ElasticSearch cluster: %s" %
(resp.status_code, action, reason))
def version(self):
"""Get version of the ElasticSearch cluster."""
if self._version is None:
self.info()
return self._version
def info(self):
"""Retrieve info about the ElasticSearch cluster."""
resp = requests.get(self._url)
self._check_response(resp)
err_msg = "Failed to retrieve info about the ElasticSearch cluster: %s"
try:
data = resp.json()
except ValueError:
LOG.debug("Return data from %s: %s" % (self._url, resp.text))
raise exceptions.RallyException(
err_msg % "The return data doesn't look like a json.")
version = data.get("version", {}).get("number")
if not version:
LOG.debug("Return data from %s: %s" % (self._url, resp.text))
raise exceptions.RallyException(
err_msg % "Failed to parse the received data.")
self._version = version
if self._version.startswith("2"):
data["version"]["build_date"] = data["version"].pop(
"build_timestamp")
return data
def push_documents(self, documents):
"""Push documents to the ElasticSearch cluster using bulk API.
:param documents: a list of documents to push
"""
LOG.debug("Pushing %s documents by chunks (up to %s documents at once)"
" to ElasticSearch." %
# dividing numbers by two, since each documents has 2 lines
# in `documents` (action and document itself).
(len(documents) / 2, self.CHUNK_LENGTH / 2))
for pos in range(0, len(documents), self.CHUNK_LENGTH):
data = "\n".join(documents[pos:pos + self.CHUNK_LENGTH]) + "\n"
raw_resp = requests.post(
self._url + "/_bulk", data=data,
headers={"Content-Type": "application/x-ndjson"}
)
self._check_response(raw_resp, action="push documents to")
LOG.debug("Successfully pushed %s documents." %
len(raw_resp.json()["items"]))
def list_indices(self):
"""List all indices."""
resp = requests.get(self._url + "/_cat/indices?v")
self._check_response(resp, "list the indices at")
return resp.text.rstrip().split(" ")
def create_index(self, name, doc_type, properties):
"""Create an index.
There are two very different ways to search strings. You can either
search whole values, that we often refer to as keyword search, or
individual tokens, that we usually refer to as full-text search.
In ElasticSearch 2.x `string` data type is used for these cases whereas
ElasticSearch 5.0 the `string` data type was replaced by two new types:
`keyword` and `text`. Since it is hard to predict the destiny of
`string` data type and support of 2 formats of input data, the
properties should be transmitted in ElasticSearch 5.x format.
"""
if self.version().startswith("2."):
properties = copy.deepcopy(properties)
for spec in properties.values():
if spec.get("type", None) == "text":
spec["type"] = "string"
elif spec.get("type", None) == "keyword":
spec["type"] = "string"
spec["index"] = "not_analyzed"
resp = requests.put(
self._url + "/%s" % name,
json={"mappings": {doc_type: {"properties": properties}}})
self._check_response(resp, "create index at")
def check_document(self, index, doc_id, doc_type="data"):
"""Check for the existence of a document.
:param index: The index of a document
:param doc_id: The ID of a document
:param doc_type: The type of a document (Defaults to data)
"""
resp = requests.head("%(url)s/%(index)s/%(type)s/%(id)s" %
{"url": self._url,
"index": index,
"type": doc_type,
"id": doc_id})
if resp.status_code == 200:
return True
elif resp.status_code == 404:
return False
else:
self._check_response(resp, "check the index at")
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

374
rally/plugins/common/exporters/elastic/exporter.py
View File

@ -12,375 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import datetime as dt
import itertools
import json
import os
from rally.plugins.task.exporters.elastic.exporter import * # noqa: F401,F403
from rally.plugins.task.exporters.elastic import exporter as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
from rally.common import validation
from rally import consts
from rally import exceptions
from rally.plugins.common.exporters.elastic import client
from rally.plugins.common.exporters.elastic import flatten
from rally.task import exporter
LOG = logging.getLogger(__name__)
@validation.configure("es_exporter_destination")
class Validator(validation.Validator):
"""Validates the destination for ElasticSearch exporter.
In case when the destination is ElasticSearch cluster, the version of it
should be 2.* or 5.*
"""
def validate(self, context, config, plugin_cls, plugin_cfg):
destination = plugin_cfg["destination"]
if destination and (not destination.startswith("http://")
and not destination.startswith("https://")):
# it is a path to a local file
return
es = client.ElasticSearchClient(destination)
try:
version = es.version()
except exceptions.RallyException as e:
# re-raise a proper exception to hide redundant traceback
self.fail(e.format_message())
if not (version.startswith("2.")
or version.startswith("5.")
or version.startswith("6.")):
self.fail("The unsupported version detected %s." % version)
@validation.add("es_exporter_destination")
@exporter.configure("elastic")
class ElasticSearchExporter(exporter.TaskExporter):
"""Exports task results to the ElasticSearch 2.x, 5.x or 6.x clusters.
The exported data includes:
* Task basic information such as title, description, status,
deployment uuid, etc.
See rally_task_v1_data index.
* Workload information such as scenario name and configuration, runner
type and configuration, time of the start load, success rate, sla
details in case of errors, etc.
See rally_workload_v1_data index.
* Separate documents for all atomic actions.
See rally_atomic_action_data_v1 index.
The destination can be a remote server. In this case specify it like:
https://elastic:changeme@example.com
Or we can dump documents to the file. The destination should look like:
/home/foo/bar.txt
In case of an empty destination, the http://localhost:9200 destination
will be used.
"""
TASK_INDEX = "rally_task_data_v1"
WORKLOAD_INDEX = "rally_workload_data_v1"
AA_INDEX = "rally_atomic_action_data_v1"
INDEX_SCHEMAS = {
TASK_INDEX: {
"task_uuid": {"type": "keyword"},
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"title": {"type": "text"},
"description": {"type": "text"},
"status": {"type": "keyword"},
"pass_sla": {"type": "boolean"},
"tags": {"type": "keyword"}
},
WORKLOAD_INDEX: {
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"scenario_name": {"type": "keyword"},
"scenario_cfg": {"type": "keyword"},
"description": {"type": "text"},
"runner_name": {"type": "keyword"},
"runner_cfg": {"type": "keyword"},
"contexts": {"type": "keyword"},
"task_uuid": {"type": "keyword"},
"subtask_uuid": {"type": "keyword"},
"started_at": {"type": "date"},
"load_duration": {"type": "long"},
"full_duration": {"type": "long"},
"pass_sla": {"type": "boolean"},
"success_rate": {"type": "float"},
"sla_details": {"type": "text"}
},
AA_INDEX: {
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"action_name": {"type": "keyword"},
"workload_uuid": {"type": "keyword"},
"scenario_cfg": {"type": "keyword"},
"contexts": {"type": "keyword"},
"runner_name": {"type": "keyword"},
"runner_cfg": {"type": "keyword"},
"success": {"type": "boolean"},
"duration": {"type": "float"},
"started_at": {"type": "date"},
"finished_at": {"type": "date"},
"parent": {"type": "keyword"},
"error": {"type": "keyword"}
}
}
def __init__(self, tasks_results, output_destination, api=None):
super(ElasticSearchExporter, self).__init__(tasks_results,
output_destination,
api=api)
self._report = []
self._remote = (
output_destination is None or (
output_destination.startswith("http://")
or self.output_destination.startswith("https://")))
if self._remote:
self._client = client.ElasticSearchClient(self.output_destination)
def _add_index(self, index, body, doc_id=None, doc_type="data"):
"""Create a document for the specified index with specified id.
:param index: The name of the index
:param body: The document. Here is the report of (sla,
scenario, iteration and atomic action)
:param doc_id: Document ID. Here we use task/subtask/workload uuid
:param doc_type: The type of document
"""
self._report.append(
json.dumps(
# use OrderedDict to make the report more unified
{"index": collections.OrderedDict([
("_index", index),
("_type", doc_type),
("_id", doc_id)])},
sort_keys=False))
self._report.append(json.dumps(body))
def _ensure_indices(self):
"""Check available indices and create require ones if they missed."""
available_index = set(self._client.list_indices())
missed_index = {self.TASK_INDEX, self.WORKLOAD_INDEX,
self.AA_INDEX} - available_index
for index in missed_index:
LOG.debug("Creating '%s' index." % index)
self._client.create_index(index, doc_type="data",
properties=self.INDEX_SCHEMAS[index])
@staticmethod
def _make_action_report(name, workload_id, workload, duration,
started_at, finished_at, parent, error):
# NOTE(andreykurilin): actually, this method just creates a dict object
# but we need to have the same format at two places, so the template
# transformed into a method.
parent = parent[0] if parent else None
return {
"deployment_uuid": workload["deployment_uuid"],
"deployment_name": workload["deployment_name"],
"action_name": name,
"workload_uuid": workload_id,
"scenario_cfg": workload["scenario_cfg"],
"contexts": workload["contexts"],
"runner_name": workload["runner_name"],
"runner_cfg": workload["runner_cfg"],
"success": not bool(error),
"duration": duration,
"started_at": started_at,
"finished_at": finished_at,
"parent": parent,
"error": error
}
def _process_atomic_actions(self, itr, workload, workload_id,
atomic_actions=None, _parent=None, _depth=0,
_cache=None):
"""Process atomic actions of an iteration
:param atomic_actions: A list with an atomic actions
:param itr: The iteration data
:param workload: The workload report
:param workload_id: The workload UUID
:param _parent: An inner parameter which is used for pointing to the
parent atomic action
:param _depth: An inner parameter which is used to mark the level of
depth while parsing atomic action children
:param _cache: An inner parameter which is used to avoid conflicts in
IDs of atomic actions of a single iteration.
"""
if _depth >= 3:
return
cache = _cache or {}
if atomic_actions is None:
atomic_actions = itr["atomic_actions"]
act_id_tmpl = "%(itr_id)s_action_%(action_name)s_%(num)s"
for i, action in enumerate(atomic_actions, 1):
cache.setdefault(action["name"], 0)
act_id = act_id_tmpl % {
"itr_id": itr["id"],
"action_name": action["name"],
"num": cache[action["name"]]}
cache[action["name"]] += 1
started_at = dt.datetime.utcfromtimestamp(action["started_at"])
finished_at = dt.datetime.utcfromtimestamp(action["finished_at"])
started_at = started_at.strftime(consts.TimeFormat.ISO8601)
finished_at = finished_at.strftime(consts.TimeFormat.ISO8601)
action_report = self._make_action_report(
name=action["name"],
workload_id=workload_id,
workload=workload,
duration=(action["finished_at"] - action["started_at"]),
started_at=started_at,
finished_at=finished_at,
parent=_parent,
error=(itr["error"] if action.get("failed", False) else None)
)
self._add_index(self.AA_INDEX, action_report,
doc_id=act_id)
self._process_atomic_actions(
atomic_actions=action["children"],
itr=itr,
workload=workload,
workload_id=workload_id,
_parent=(act_id, action_report),
_depth=(_depth + 1),
_cache=cache)
if itr["error"] and (
# the case when it is a top level of the scenario and the
# first fails the item which is not wrapped by AtomicTimer
(not _parent and not atomic_actions)
# the case when it is a top level of the scenario and and
# the item fails after some atomic actions completed
or (not _parent and atomic_actions
and not atomic_actions[-1].get("failed", False))):
act_id = act_id_tmpl % {
"itr_id": itr["id"],
"action_name": "no-name-action",
"num": 0}
# Since the action had not be wrapped by AtomicTimer, we cannot
# make any assumption about it's duration (start_time) so let's use
# finished_at timestamp of iteration with 0 duration
timestamp = (itr["timestamp"] + itr["duration"]
+ itr["idle_duration"])
timestamp = dt.datetime.utcfromtimestamp(timestamp)
timestamp = timestamp.strftime(consts.TimeFormat.ISO8601)
action_report = self._make_action_report(
name="no-name-action",
workload_id=workload_id,
workload=workload,
duration=0,
started_at=timestamp,
finished_at=timestamp,
parent=_parent,
error=itr["error"]
)
self._add_index(self.AA_INDEX, action_report, doc_id=act_id)
def generate(self):
if self._remote:
self._ensure_indices()
for task in self.tasks_results:
if self._remote:
if self._client.check_document(self.TASK_INDEX, task["uuid"]):
raise exceptions.RallyException(
"Failed to push the task %s to the ElasticSearch "
"cluster. The document with such UUID already exists" %
task["uuid"])
task_report = {
"task_uuid": task["uuid"],
"deployment_uuid": task["env_uuid"],
"deployment_name": task["env_name"],
"title": task["title"],
"description": task["description"],
"status": task["status"],
"pass_sla": task["pass_sla"],
"tags": task["tags"]
}
self._add_index(self.TASK_INDEX, task_report,
doc_id=task["uuid"])
# NOTE(andreykurilin): The subtasks do not have much logic now, so
# there is no reason to save the info about them.
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
durations = workload["statistics"]["durations"]
success_rate = durations["total"]["data"]["success"]
if success_rate == "n/a":
success_rate = 0.0
else:
# cut the % char and transform to the float value
success_rate = float(success_rate[:-1]) / 100.0
started_at = workload["start_time"]
if started_at:
started_at = dt.datetime.utcfromtimestamp(started_at)
started_at = started_at.strftime(consts.TimeFormat.ISO8601)
workload_report = {
"task_uuid": workload["task_uuid"],
"subtask_uuid": workload["subtask_uuid"],
"deployment_uuid": task["env_uuid"],
"deployment_name": task["env_name"],
"scenario_name": workload["name"],
"scenario_cfg": flatten.transform(workload["args"]),
"description": workload["description"],
"runner_name": workload["runner_type"],
"runner_cfg": flatten.transform(workload["runner"]),
"contexts": flatten.transform(workload["contexts"]),
"started_at": started_at,
"load_duration": workload["load_duration"],
"full_duration": workload["full_duration"],
"pass_sla": workload["pass_sla"],
"success_rate": success_rate,
"sla_details": [s["detail"]
for s in workload["sla_results"]["sla"]
if not s["success"]]}
# do we need to store hooks ?!
self._add_index(self.WORKLOAD_INDEX, workload_report,
doc_id=workload["uuid"])
# Iterations
for idx, itr in enumerate(workload.get("data", []), 1):
itr["id"] = "%(uuid)s_iter_%(num)s" % {
"uuid": workload["uuid"],
"num": str(idx)}
self._process_atomic_actions(
itr=itr,
workload=workload_report,
workload_id=workload["uuid"])
if self._remote:
LOG.debug("The info of ElasticSearch cluster to which the results "
"will be exported: %s" % self._client.info())
self._client.push_documents(self._report)
msg = ("Successfully exported results to ElasticSearch at url "
"'%s'" % self.output_destination)
return {"print": msg}
else:
# a new line is required in the end of the file.
report = "\n".join(self._report) + "\n"
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

55
rally/plugins/common/exporters/elastic/flatten.py
View File

@ -12,54 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.exporters.elastic.flatten import * # noqa: F401,F403
from rally.plugins.task.exporters.elastic import flatten as _new
def _join_keys(first, second):
if not second:
return first
elif second.startswith("["):
return "%s%s" % (first, second)
else:
return "%s.%s" % (first, second)
# import it as last item to be sure that we use the right module
from rally.common import logging
def _process(obj):
if isinstance(obj, (str, bytes)):
yield "", obj
elif isinstance(obj, dict):
for first, tmp_value in obj.items():
for second, value in _process(tmp_value):
yield _join_keys(first, second), value
elif isinstance(obj, (list, tuple)):
for i, tmp_value in enumerate(obj):
for second, value in _process(tmp_value):
yield _join_keys("[%s]" % i, second), value
else:
try:
yield "", "%s" % obj
except Exception:
raise ValueError("Cannot transform obj of '%s' type to flatten "
"structure." % type(obj))
def transform(obj):
"""Transform object to a flatten structure.
Example:
IN:
{"foo": ["xxx", "yyy", {"bar": {"zzz": ["Hello", "World!"]}}]}
OUTPUT:
[
"foo[0]=xxx",
"foo[1]=yyy",
"foo[2].bar.zzz[0]=Hello",
"foo[2].bar.zzz[1]=World!"
]
"""
result = []
for key, value in _process(obj):
if key:
result.append("%s=%s" % (key, value))
else:
result.append(value)
return sorted(result)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

46
rally/plugins/common/exporters/html.py
View File

@ -12,45 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import os
from rally.plugins.task.exporters.html import * # noqa: F401,F403
from rally.plugins.task.exporters import html as _new
from rally.task import exporter
from rally.task.processing import plot
# import it as last item to be sure that we use the right module
from rally.common import logging
@exporter.configure("html")
class HTMLExporter(exporter.TaskExporter):
"""Generates task report in HTML format."""
INCLUDE_LIBS = False
def _generate_results(self):
results = []
processed_names = {}
for task in self.tasks_results:
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
if workload["name"] in processed_names:
processed_names[workload["name"]] += 1
workload["position"] = processed_names[workload["name"]]
else:
processed_names[workload["name"]] = 0
results.append(task)
return results
def generate(self):
report = plot.plot(self._generate_results(),
include_libs=self.INCLUDE_LIBS)
if self.output_destination:
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": report}
@exporter.configure("html-static")
class HTMLStaticExporter(HTMLExporter):
"""Generates task report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

113
rally/plugins/common/exporters/json_exporter.py
View File

@ -12,112 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import datetime as dt
import json
from rally.plugins.task.exporters.json_exporter import * # noqa: F401,F403
from rally.plugins.task.exporters import json_exporter as _new
from rally.common import version as rally_version
from rally.task import exporter
TIMEFORMAT = "%Y-%m-%dT%H:%M:%S"
# import it as last item to be sure that we use the right module
from rally.common import logging
@exporter.configure("json")
class JSONExporter(exporter.TaskExporter):
"""Generates task report in JSON format."""
# Revisions:
# 1.0 - the json report v1
# 1.1 - add `contexts_results` key with contexts execution results of
# workloads.
# 1.2 - add `env_uuid` and `env_uuid` which represent environment name
# and UUID where task was executed
REVISION = "1.2"
def _generate_tasks(self):
tasks = []
for task in self.tasks_results:
subtasks = []
for subtask in task["subtasks"]:
workloads = []
for workload in subtask["workloads"]:
hooks = [{
"config": {"action": dict([h["config"]["action"]]),
"trigger": dict([h["config"]["trigger"]]),
"description": h["config"]["description"]},
"results": h["results"],
"summary": h["summary"], } for h in workload["hooks"]]
workloads.append(
collections.OrderedDict(
[("uuid", workload["uuid"]),
("description", workload["description"]),
("runner", {
workload["runner_type"]: workload["runner"]}),
("hooks", hooks),
("scenario", {
workload["name"]: workload["args"]}),
("min_duration", workload["min_duration"]),
("max_duration", workload["max_duration"]),
("start_time", workload["start_time"]),
("load_duration", workload["load_duration"]),
("full_duration", workload["full_duration"]),
("statistics", workload["statistics"]),
("data", workload["data"]),
("failed_iteration_count",
workload["failed_iteration_count"]),
("total_iteration_count",
workload["total_iteration_count"]),
("created_at", workload["created_at"]),
("updated_at", workload["updated_at"]),
("contexts", workload["contexts"]),
("contexts_results",
workload["contexts_results"]),
("position", workload["position"]),
("pass_sla", workload["pass_sla"]),
("sla_results", workload["sla_results"]),
("sla", workload["sla"])]
)
)
subtasks.append(
collections.OrderedDict(
[("uuid", subtask["uuid"]),
("title", subtask["title"]),
("description", subtask["description"]),
("status", subtask["status"]),
("created_at", subtask["created_at"]),
("updated_at", subtask["updated_at"]),
("sla", subtask["sla"]),
("workloads", workloads)]
)
)
tasks.append(
collections.OrderedDict(
[("uuid", task["uuid"]),
("title", task["title"]),
("description", task["description"]),
("status", task["status"]),
("tags", task["tags"]),
("env_uuid", task.get("env_uuid", "n\a")),
("env_name", task.get("env_name", "n\a")),
("created_at", task["created_at"]),
("updated_at", task["updated_at"]),
("pass_sla", task["pass_sla"]),
("subtasks", subtasks)]
)
)
return tasks
def generate(self):
results = {"info": {"rally_version": rally_version.version_string(),
"generated_at": dt.datetime.strftime(
dt.datetime.utcnow(), TIMEFORMAT),
"format_version": self.REVISION},
"tasks": self._generate_tasks()}
results = json.dumps(results, sort_keys=False, indent=4)
if self.output_destination:
return {"files": {self.output_destination: results},
"open": "file://" + self.output_destination}
else:
return {"print": results}
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

86
rally/plugins/common/exporters/junit.py
View File

@ -12,85 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime as dt
import itertools
import os
from rally.plugins.task.exporters.junit import * # noqa: F401,F403
from rally.plugins.task.exporters import junit as _new
from rally.common.io import junit
from rally.task import exporter
# import it as last item to be sure that we use the right module
from rally.common import logging
@exporter.configure("junit-xml")
class JUnitXMLExporter(exporter.TaskExporter):
"""Generates task report in JUnit-XML format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: xml
<testsuites>
<!--Report is generated by Rally 0.10.0 at 2017-06-04T05:14:00-->
<testsuite id="task-uu-ii-dd"
errors="0"
failures="1"
skipped="0"
tests="2"
time="75.0"
timestamp="2017-06-04T05:14:00">
<testcase classname="CinderVolumes"
name="list_volumes"
id="workload-1-uuid"
time="29.9695231915"
timestamp="2017-06-04T05:14:44" />
<testcase classname="NovaServers"
name="list_keypairs"
id="workload-2-uuid"
time="5"
timestamp="2017-06-04T05:15:15">
<failure>ooops</failure>
</testcase>
</testsuite>
</testsuites>
"""
def generate(self):
root = junit.JUnitXML()
for t in self.tasks_results:
created_at = dt.datetime.strptime(t["created_at"],
"%Y-%m-%dT%H:%M:%S")
updated_at = dt.datetime.strptime(t["updated_at"],
"%Y-%m-%dT%H:%M:%S")
test_suite = root.add_test_suite(
id=t["uuid"],
time="%.2f" % (updated_at - created_at).total_seconds(),
timestamp=t["created_at"]
)
for workload in itertools.chain(
*[s["workloads"] for s in t["subtasks"]]):
class_name, name = workload["name"].split(".", 1)
test_case = test_suite.add_test_case(
id=workload["uuid"],
time="%.2f" % workload["full_duration"],
classname=class_name,
name=name,
timestamp=workload["created_at"]
)
if not workload["pass_sla"]:
details = "\n".join(
[s["detail"]
for s in workload["sla_results"]["sla"]
if not s["success"]]
)
test_case.mark_as_failed(details)
raw_report = root.to_string()
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": raw_report}
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

30
rally/plugins/common/exporters/trends.py
View File

@ -1,4 +1,3 @@
# Copyright 2018: ZTE Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,28 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from rally.plugins.task.exporters.trends import * # noqa: F401,F403
from rally.plugins.task.exporters import trends as _new
from rally.task import exporter
from rally.task.processing import plot
# import it as last item to be sure that we use the right module
from rally.common import logging
@exporter.configure("trends-html")
class TrendsExporter(exporter.TaskExporter):
"""Generates task trends report in HTML format."""
INCLUDE_LIBS = False
def generate(self):
report = plot.trends(self.tasks_results, self.INCLUDE_LIBS)
if self.output_destination:
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": report}
@exporter.configure("trends-html-static")
class TrendsStaticExport(TrendsExporter):
"""Generates task trends report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

56
rally/plugins/common/hook/sys_call.py
View File

@ -1,4 +1,3 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,56 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import shlex
import subprocess
from rally.plugins.task.hooks.sys_call import * # noqa: F401,F403
from rally.plugins.task.hooks import sys_call as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
from rally import consts
from rally import exceptions
from rally.task import hook
LOG = logging.getLogger(__name__)
@hook.configure(name="sys_call")
class SysCallHook(hook.HookAction):
"""Performs system call."""
CONFIG_SCHEMA = {
"$schema": consts.JSON_SCHEMA,
"type": "string",
"description": "Command to execute."
}
def run(self):
LOG.debug("sys_call hook: Running command %s" % self.config)
proc = subprocess.Popen(shlex.split(self.config),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
out, err = proc.communicate()
LOG.debug("sys_call hook: Command %s returned %s"
% (self.config, proc.returncode))
if proc.returncode:
self.set_error(
exception_name="n/a", # no exception class
description="Subprocess returned %s" % proc.returncode,
details=(err or "stdout: %s" % out))
# NOTE(amaretskiy): Try to load JSON for charts,
# otherwise save output as-is
try:
output = json.loads(out)
for arg in ("additive", "complete"):
for out_ in output.get(arg, []):
self.add_output(**{arg: out_})
except (TypeError, ValueError, exceptions.RallyException):
self.add_output(
complete={"title": "System call",
"chart_plugin": "TextArea",
"description": "Args: %s" % self.config,
"data": ["RetCode: %i" % proc.returncode,
"StdOut: %s" % (out or "(empty)"),
"StdErr: %s" % (err or "(empty)")]})
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

66
rally/plugins/common/hook/triggers/event.py
View File

@ -1,4 +1,3 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,62 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally import consts
from rally.task import hook
from rally.plugins.task.hook_triggers.event import * # noqa: F401,F403
from rally.plugins.task.hook_triggers import event as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
@hook.configure(name="event")
class EventTrigger(hook.HookTrigger):
"""Triggers hook on specified event and list of values."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"oneOf": [
{
"description": "Triage hook based on specified seconds after "
"start of workload.",
"properties": {
"unit": {"enum": ["time"]},
"at": {
"type": "array",
"minItems": 1,
"uniqueItems": True,
"items": {
"type": "integer",
"minimum": 0
}
},
},
"required": ["unit", "at"],
"additionalProperties": False,
},
{
"description": "Triage hook based on specific iterations.",
"properties": {
"unit": {"enum": ["iteration"]},
"at": {
"type": "array",
"minItems": 1,
"uniqueItems": True,
"items": {
"type": "integer",
"minimum": 1,
}
},
},
"required": ["unit", "at"],
"additionalProperties": False,
},
]
}
def get_listening_event(self):
return self.config["unit"]
def on_event(self, event_type, value=None):
if not (event_type == self.get_listening_event()
and value in self.config["at"]):
# do nothing
return
super(EventTrigger, self).on_event(event_type, value)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

61
rally/plugins/common/hook/triggers/periodic.py
View File

@ -1,4 +1,3 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,57 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally import consts
from rally.task import hook
from rally.plugins.task.hook_triggers.periodic import * # noqa: F401,F403
from rally.plugins.task.hook_triggers import periodic as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
@hook.configure(name="periodic")
class PeriodicTrigger(hook.HookTrigger):
"""Periodically triggers hook with specified range and step."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"oneOf": [
{
"description": "Periodically triage hook based on elapsed time"
" after start of workload.",
"properties": {
"unit": {"enum": ["time"]},
"start": {"type": "integer", "minimum": 0},
"end": {"type": "integer", "minimum": 1},
"step": {"type": "integer", "minimum": 1},
},
"required": ["unit", "step"],
"additionalProperties": False,
},
{
"description": "Periodically triage hook based on iterations.",
"properties": {
"unit": {"enum": ["iteration"]},
"start": {"type": "integer", "minimum": 1},
"end": {"type": "integer", "minimum": 1},
"step": {"type": "integer", "minimum": 1},
},
"required": ["unit", "step"],
"additionalProperties": False,
},
]
}
def __init__(self, context, task, hook_cls):
super(PeriodicTrigger, self).__init__(context, task, hook_cls)
self.config.setdefault(
"start", 0 if self.config["unit"] == "time" else 1)
self.config.setdefault("end", float("Inf"))
def get_listening_event(self):
return self.config["unit"]
def on_event(self, event_type, value=None):
if not (event_type == self.get_listening_event()
and self.config["start"] <= value <= self.config["end"]
and (value - self.config["start"]) % self.config["step"] == 0):
# do nothing
return
super(PeriodicTrigger, self).on_event(event_type, value)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

332
rally/plugins/common/runners/constant.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,330 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import multiprocessing
import queue as Queue
import threading
import time
from rally.plugins.task.runners.constant import * # noqa: F401,F403
from rally.plugins.task.runners import constant as _new
from rally.common import utils
from rally.common import validation
from rally import consts
from rally.task import runner
# import it as last item to be sure that we use the right module
from rally.common import logging
def _worker_process(queue, iteration_gen, timeout, concurrency, times,
duration, context, cls, method_name, args, event_queue,
aborted, info):
"""Start the scenario within threads.
Spawn threads to support scenario execution.
Scenario is ran for a fixed number of times if times is specified
Scenario is ran for fixed duration if duration is specified.
This generates a constant load on the cloud under test by executing each
scenario iteration without pausing between iterations. Each thread runs
the scenario method once with passed scenario arguments and context.
After execution the result is appended to the queue.
:param queue: queue object to append results
:param iteration_gen: next iteration number generator
:param timeout: operation's timeout
:param concurrency: number of concurrently running scenario iterations
:param times: total number of scenario iterations to be run
:param duration: total duration in seconds of the run
:param context: scenario context object
:param cls: scenario class
:param method_name: scenario method name
:param args: scenario args
:param event_queue: queue object to append events
:param aborted: multiprocessing.Event that aborts load generation if
the flag is set
:param info: info about all processes count and counter of launched process
"""
def _to_be_continued(iteration, current_duration, aborted, times=None,
duration=None):
if times is not None:
return iteration < times and not aborted.is_set()
elif duration is not None:
return current_duration < duration and not aborted.is_set()
else:
return False
if times is None and duration is None:
raise ValueError("times or duration must be specified")
pool = collections.deque()
alive_threads_in_pool = 0
finished_threads_in_pool = 0
runner._log_worker_info(times=times, duration=duration,
concurrency=concurrency, timeout=timeout, cls=cls,
method_name=method_name, args=args)
if timeout:
timeout_queue = Queue.Queue()
collector_thr_by_timeout = threading.Thread(
target=utils.timeout_thread,
args=(timeout_queue, )
)
collector_thr_by_timeout.start()
iteration = next(iteration_gen)
start_time = time.time()
# NOTE(msimonin): keep the previous behaviour
# > when duration is 0, scenario executes exactly 1 time
current_duration = -1
while _to_be_continued(iteration, current_duration, aborted,
times=times, duration=duration):
scenario_context = runner._get_scenario_context(iteration, context)
worker_args = (
queue, cls, method_name, scenario_context, args, event_queue)
thread = threading.Thread(target=runner._worker_thread,
args=worker_args)
thread.start()
if timeout:
timeout_queue.put((thread, time.time() + timeout))
pool.append(thread)
alive_threads_in_pool += 1
while alive_threads_in_pool == concurrency:
prev_finished_threads_in_pool = finished_threads_in_pool
finished_threads_in_pool = 0
for t in pool:
if not t.is_alive():
finished_threads_in_pool += 1
alive_threads_in_pool -= finished_threads_in_pool
alive_threads_in_pool += prev_finished_threads_in_pool
if alive_threads_in_pool < concurrency:
# NOTE(boris-42): cleanup pool array. This is required because
# in other case array length will be equal to times which
# is unlimited big
while pool and not pool[0].is_alive():
pool.popleft().join()
finished_threads_in_pool -= 1
break
# we should wait to not create big noise with these checks
time.sleep(0.001)
iteration = next(iteration_gen)
current_duration = time.time() - start_time
# Wait until all threads are done
while pool:
pool.popleft().join()
if timeout:
timeout_queue.put((None, None,))
collector_thr_by_timeout.join()
@validation.configure("check_constant")
class CheckConstantValidator(validation.Validator):
"""Additional schema validation for constant runner"""
def validate(self, context, config, plugin_cls, plugin_cfg):
if plugin_cfg.get("concurrency", 1) > plugin_cfg.get("times", 1):
return self.fail(
"Parameter 'concurrency' means a number of parallel "
"executions of iterations. Parameter 'times' means total "
"number of iteration executions. It is redundant "
"(and restricted) to have number of parallel iterations "
"bigger then total number of iterations.")
@validation.add("check_constant")
@runner.configure(name="constant")
class ConstantScenarioRunner(runner.ScenarioRunner):
"""Creates constant load executing a scenario a specified number of times.
This runner will place a constant load on the cloud under test by
executing each scenario iteration without pausing between iterations
up to the number of times specified in the scenario config.
The concurrency parameter of the scenario config controls the
number of concurrent iterations which execute during a single
scenario in order to simulate the activities of multiple users
placing load on the cloud under test.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"concurrency": {
"type": "integer",
"minimum": 1,
"description": "The number of parallel iteration executions."
},
"times": {
"type": "integer",
"minimum": 1,
"description": "Total number of iteration executions."
},
"timeout": {
"type": "number",
"description": "Operation's timeout."
},
"max_cpu_count": {
"type": "integer",
"minimum": 1,
"description": "The maximum number of processes to create load"
" from."
}
},
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
This method generates a constant load on the cloud under test by
executing each scenario iteration using a pool of processes without
pausing between iterations up to the number of times specified
in the scenario config.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
timeout = self.config.get("timeout", 0) # 0 means no timeout
times = self.config.get("times", 1)
concurrency = self.config.get("concurrency", 1)
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
processes_to_start = min(max_cpu_used, times, concurrency)
concurrency_per_worker, concurrency_overhead = divmod(
concurrency, processes_to_start)
self._log_debug_info(times=times, concurrency=concurrency,
timeout=timeout, max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(concurrency_overhead):
while True:
yield (result_queue, iteration_gen, timeout,
concurrency_per_worker + (concurrency_overhead and 1),
times, None, context, cls, method_name, args,
event_queue, self.aborted)
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
@runner.configure(name="constant_for_duration")
class ConstantForDurationScenarioRunner(runner.ScenarioRunner):
"""Creates constant load executing a scenario for an interval of time.
This runner will place a constant load on the cloud under test by
executing each scenario iteration without pausing between iterations
until a specified interval of time has elapsed.
The concurrency parameter of the scenario config controls the
number of concurrent iterations which execute during a single
sceanario in order to simulate the activities of multiple users
placing load on the cloud under test.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"concurrency": {
"type": "integer",
"minimum": 1,
"description": "The number of parallel iteration executions."
},
"duration": {
"type": "number",
"minimum": 0.0,
"description": "The number of seconds during which to generate"
" a load. If the duration is 0, the scenario"
" will run once per parallel execution."
},
"timeout": {
"type": "number",
"minimum": 1,
"description": "Operation's timeout."
}
},
"required": ["duration"],
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
This method generates a constant load on the cloud under test by
executing each scenario iteration using a pool of processes without
pausing between iterations up to the number of times specified
in the scenario config.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
timeout = self.config.get("timeout", 600)
duration = self.config.get("duration", 0)
concurrency = self.config.get("concurrency", 1)
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
processes_to_start = min(max_cpu_used, concurrency)
concurrency_per_worker, concurrency_overhead = divmod(
concurrency, processes_to_start)
self._log_debug_info(duration=duration, concurrency=concurrency,
timeout=timeout, max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(concurrency_overhead):
while True:
yield (result_queue, iteration_gen, timeout,
concurrency_per_worker + (concurrency_overhead and 1),
None, duration, context, cls, method_name, args,
event_queue, self.aborted)
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

285
rally/plugins/common/runners/rps.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,285 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import multiprocessing
import queue as Queue
import threading
import time
from rally.plugins.task.runners.rps import * # noqa: F401,F403
from rally.plugins.task.runners import rps as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
from rally.common import utils
from rally.common import validation
from rally import consts
from rally.task import runner
LOG = logging.getLogger(__name__)
def _worker_process(queue, iteration_gen, timeout, times, max_concurrent,
context, cls, method_name, args, event_queue, aborted,
runs_per_second, rps_cfg, processes_to_start, info):
"""Start scenario within threads.
Spawn N threads per second. Each thread runs the scenario once, and appends
result to queue. A maximum of max_concurrent threads will be ran
concurrently.
:param queue: queue object to append results
:param iteration_gen: next iteration number generator
:param timeout: operation's timeout
:param times: total number of scenario iterations to be run
:param max_concurrent: maximum worker concurrency
:param context: scenario context object
:param cls: scenario class
:param method_name: scenario method name
:param args: scenario args
:param aborted: multiprocessing.Event that aborts load generation if
the flag is set
:param runs_per_second: function that should return desired rps value
:param rps_cfg: rps section from task config
:param processes_to_start: int, number of started processes for scenario
execution
:param info: info about all processes count and counter of runned process
"""
pool = collections.deque()
if isinstance(rps_cfg, dict):
rps = rps_cfg["start"]
else:
rps = rps_cfg
sleep = 1.0 / rps
runner._log_worker_info(times=times, rps=rps, timeout=timeout,
cls=cls, method_name=method_name, args=args)
time.sleep(
(sleep * info["processes_counter"]) / info["processes_to_start"])
start = time.time()
timeout_queue = Queue.Queue()
if timeout:
collector_thr_by_timeout = threading.Thread(
target=utils.timeout_thread,
args=(timeout_queue, )
)
collector_thr_by_timeout.start()
i = 0
while i < times and not aborted.is_set():
scenario_context = runner._get_scenario_context(next(iteration_gen),
context)
worker_args = (
queue, cls, method_name, scenario_context, args, event_queue)
thread = threading.Thread(target=runner._worker_thread,
args=worker_args)
i += 1
thread.start()
if timeout:
timeout_queue.put((thread, time.time() + timeout))
pool.append(thread)
time_gap = time.time() - start
real_rps = i / time_gap if time_gap else "Infinity"
LOG.debug(
"Worker: %s rps: %s (requested rps: %s)" %
(i, real_rps, runs_per_second(rps_cfg, start, processes_to_start)))
# try to join latest thread(s) until it finished, or until time to
# start new thread (if we have concurrent slots available)
while i / (time.time() - start) > runs_per_second(
rps_cfg, start, processes_to_start) or (
len(pool) >= max_concurrent):
if pool:
pool[0].join(0.001)
if not pool[0].is_alive():
pool.popleft()
else:
time.sleep(0.001)
while pool:
pool.popleft().join()
if timeout:
timeout_queue.put((None, None,))
collector_thr_by_timeout.join()
@validation.configure("check_rps")
class CheckPRSValidator(validation.Validator):
"""Additional schema validation for rps runner"""
def validate(self, context, config, plugin_cls, plugin_cfg):
if isinstance(plugin_cfg["rps"], dict):
if plugin_cfg["rps"]["end"] < plugin_cfg["rps"]["start"]:
msg = "rps end value must not be less than rps start value."
return self.fail(msg)
@validation.add("check_rps")
@runner.configure(name="rps")
class RPSScenarioRunner(runner.ScenarioRunner):
"""Scenario runner that does the job with specified frequency.
Every single scenario iteration is executed with specified frequency
(runs per second) in a pool of processes. The scenario will be
launched for a fixed number of times in total (specified in the config).
An example of a rps scenario is booting 1 VM per second. This
execution type is thus very helpful in understanding the maximal load that
a certain cloud can handle.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"times": {
"type": "integer",
"minimum": 1
},
"rps": {
"anyOf": [
{
"description": "Generate constant requests per second "
"during the whole workload.",
"type": "number",
"exclusiveMinimum": 0,
"minimum": 0
},
{
"type": "object",
"description": "Increase requests per second for "
"specified value each time after a "
"certain number of seconds.",
"properties": {
"start": {
"type": "number",
"minimum": 1
},
"end": {
"type": "number",
"minimum": 1
},
"step": {
"type": "number",
"minimum": 1
},
"duration": {
"type": "number",
"minimum": 1
}
},
"additionalProperties": False,
"required": ["start", "end", "step"]
}
],
},
"timeout": {
"type": "number",
},
"max_concurrency": {
"type": "integer",
"minimum": 1
},
"max_cpu_count": {
"type": "integer",
"minimum": 1
}
},
"required": ["times", "rps"],
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
Every single scenario iteration is executed with specified
frequency (runs per second) in a pool of processes. The scenario is
launched for a fixed number of times in total (specified in the
config).
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: Context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config["times"]
timeout = self.config.get("timeout", 0) # 0 means no timeout
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
def runs_per_second(rps_cfg, start_timer, number_of_processes):
"""At the given second return desired rps."""
if not isinstance(rps_cfg, dict):
return float(rps_cfg) / number_of_processes
stage_order = (time.time() - start_timer) / rps_cfg.get(
"duration", 1) - 1
rps = (float(rps_cfg["start"] + rps_cfg["step"] * stage_order)
/ number_of_processes)
return min(rps, float(rps_cfg["end"]))
processes_to_start = min(max_cpu_used, times,
self.config.get("max_concurrency", times))
times_per_worker, times_overhead = divmod(times, processes_to_start)
# Determine concurrency per worker
concurrency_per_worker, concurrency_overhead = divmod(
self.config.get("max_concurrency", times), processes_to_start)
self._log_debug_info(times=times, timeout=timeout,
max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
times_per_worker=times_per_worker,
times_overhead=times_overhead,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(times_overhead, concurrency_overhead):
"""Generate arguments for process worker.
Remainder of threads per process division is distributed to
process workers equally - one thread per each process worker
until the remainder equals zero. The same logic is applied
to concurrency overhead.
:param times_overhead: remaining number of threads to be
distributed to workers
:param concurrency_overhead: remaining number of maximum
concurrent threads to be
distributed to workers
"""
while True:
yield (
result_queue, iteration_gen, timeout,
times_per_worker + (times_overhead and 1),
concurrency_per_worker + (concurrency_overhead and 1),
context, cls, method_name, args, event_queue,
self.aborted, runs_per_second, self.config["rps"],
processes_to_start
)
if times_overhead:
times_overhead -= 1
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(times_overhead, concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

69
rally/plugins/common/runners/serial.py
View File

@ -1,4 +1,3 @@
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,65 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.common import utils as rutils
from rally import consts
from rally.task import runner
from rally.plugins.task.runners.serial import * # noqa: F401,F403
from rally.plugins.task.runners import serial as _new
# import it as last item to be sure that we use the right module
from rally.common import logging
@runner.configure(name="serial")
class SerialScenarioRunner(runner.ScenarioRunner):
"""Scenario runner that executes scenarios serially.
Unlike scenario runners that execute in parallel, the serial scenario
runner executes scenarios one-by-one in the same python interpreter process
as Rally. This allows you to execute scenario without introducing
any concurrent operations as well as interactively debug the scenario
from the same command that you use to start Rally.
"""
# NOTE(mmorais): additionalProperties is set True to allow switching
# between parallel and serial runners by modifying only *type* property
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"times": {
"type": "integer",
"minimum": 1
}
},
"additionalProperties": True
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
The scenario iterations are executed one-by-one in the same python
interpreter process as Rally. This allows you to execute
scenario without introducing any concurrent operations as well as
interactively debug the scenario from the same command that you use
to start Rally.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config.get("times", 1)
event_queue = rutils.DequeAsQueue(self.event_queue)
for i in range(times):
if self.aborted.is_set():
break
result = runner._run_scenario_once(
cls, method_name, runner._get_scenario_context(i, context),
args, event_queue)
self._send_result(result)
self._flush_results()
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

50
rally/plugins/common/scenarios/requests/http_requests.py
View File

@ -1,3 +1,5 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -10,47 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import random
from rally.plugins.task.scenarios.requests.http_requests import * # noqa: F401,F403,E501
from rally.plugins.task.scenarios.requests import http_requests as _new
from rally.plugins.common.scenarios.requests import utils
from rally.task import scenario
# import it as last item to be sure that we use the right module
from rally.common import logging
"""Scenarios for HTTP requests."""
@scenario.configure(name="HttpRequests.check_request")
class HttpRequestsCheckRequest(utils.RequestScenario):
def run(self, url, method, status_code, **kwargs):
"""Standard way for testing web services using HTTP requests.
This scenario is used to make request and check it with expected
Response.
:param url: url for the Request object
:param method: method for the Request object
:param status_code: expected response code
:param kwargs: optional additional request parameters
"""
self._check_request(url, method, status_code, **kwargs)
@scenario.configure(name="HttpRequests.check_random_request")
class HttpRequestsCheckRandomRequest(utils.RequestScenario):
def run(self, requests, status_code):
"""Executes random HTTP requests from provided list.
This scenario takes random url from list of requests, and raises
exception if the response is not the expected response.
:param requests: List of request dicts
:param status_code: Expected Response Code it will
be used only if we doesn't specified it in request proper
"""
request = random.choice(requests)
request.setdefault("status_code", status_code)
self._check_request(**request)
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

32
rally/plugins/common/scenarios/requests/utils.py
View File

@ -1,3 +1,5 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -10,29 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import requests
from rally.plugins.task.scenarios.requests.utils import * # noqa: F401,F403
from rally.plugins.task.scenarios.requests import utils as _new
from rally.task import atomic
from rally.task import scenario
# import it as last item to be sure that we use the right module
from rally.common import logging
class RequestScenario(scenario.Scenario):
"""Base class for Request scenarios with basic atomic actions."""
@atomic.action_timer("requests.check_request")
def _check_request(self, url, method, status_code, **kwargs):
"""Compare request status code with specified code
:param status_code: Expected status code of request
:param url: Uniform resource locator
:param method: Type of request method (GET | POST ..)
:param kwargs: Optional additional request parameters
:raises ValueError: if return http status code
not equal to expected status code
"""
resp = requests.request(method, url, **kwargs)
if status_code != resp.status_code:
error_msg = "Expected HTTP request code is `%s` actual `%s`"
raise ValueError(
error_msg % (status_code, resp.status_code))
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

57
rally/plugins/common/sla/failure_rate.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,55 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.failure_rate import * # noqa: F401,F403
from rally.plugins.task.sla import failure_rate as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally import consts
from rally.task import sla
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="failure_rate")
class FailureRate(sla.SLA):
"""Failure rate minimum and maximum in percents."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"min": {"type": "number", "minimum": 0.0, "maximum": 100.0},
"max": {"type": "number", "minimum": 0.0, "maximum": 100.0}
},
"minProperties": 1,
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(FailureRate, self).__init__(criterion_value)
self.min_percent = self.criterion_value.get("min", 0)
self.max_percent = self.criterion_value.get("max", 100)
self.errors = 0
self.total = 0
self.error_rate = 0.0
def add_iteration(self, iteration):
self.total += 1
if iteration["error"]:
self.errors += 1
self.error_rate = self.errors * 100.0 / self.total
self.success = self.min_percent <= self.error_rate <= self.max_percent
return self.success
def merge(self, other):
self.total += other.total
self.errors += other.errors
if self.total:
self.error_rate = self.errors * 100.0 / self.total
self.success = self.min_percent <= self.error_rate <= self.max_percent
return self.success
def details(self):
return ("Failure rate criteria %.2f%% <= %.2f%% <= %.2f%% - %s" %
(self.min_percent, self.error_rate,
self.max_percent, self.status()))
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

43
rally/plugins/common/sla/iteration_time.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,41 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.iteration_time import * # noqa: F401,F403
from rally.plugins.task.sla import iteration_time as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally import consts
from rally.task import sla
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="max_seconds_per_iteration")
class IterationTime(sla.SLA):
"""Maximum time for one iteration in seconds."""
CONFIG_SCHEMA = {
"type": "number",
"$schema": consts.JSON_SCHEMA7,
"minimum": 0.0,
"exclusiveMinimum": 0.0}
def __init__(self, criterion_value):
super(IterationTime, self).__init__(criterion_value)
self.max_iteration_time = 0.0
def add_iteration(self, iteration):
if iteration["duration"] > self.max_iteration_time:
self.max_iteration_time = iteration["duration"]
self.success = self.max_iteration_time <= self.criterion_value
return self.success
def merge(self, other):
if other.max_iteration_time > self.max_iteration_time:
self.max_iteration_time = other.max_iteration_time
self.success = self.max_iteration_time <= self.criterion_value
return self.success
def details(self):
return ("Maximum seconds per iteration %.2fs <= %.2fs - %s" %
(self.max_iteration_time, self.criterion_value, self.status()))
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

46
rally/plugins/common/sla/max_average_duration.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,44 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.max_average_duration import * # noqa: F401,F403
from rally.plugins.task.sla import max_average_duration as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="max_avg_duration")
class MaxAverageDuration(sla.SLA):
"""Maximum average duration of one iteration in seconds."""
CONFIG_SCHEMA = {
"type": "number",
"$schema": consts.JSON_SCHEMA7,
"exclusiveMinimum": 0.0
}
def __init__(self, criterion_value):
super(MaxAverageDuration, self).__init__(criterion_value)
self.avg = 0.0
self.avg_comp = streaming_algorithms.MeanComputation()
def add_iteration(self, iteration):
if not iteration.get("error"):
self.avg_comp.add(iteration["duration"])
self.avg = self.avg_comp.result()
self.success = self.avg <= self.criterion_value
return self.success
def merge(self, other):
self.avg_comp.merge(other.avg_comp)
self.avg = self.avg_comp.result() or 0.0
self.success = self.avg <= self.criterion_value
return self.success
def details(self):
return ("Average duration of one iteration %.2fs <= %.2fs - %s" %
(self.avg, self.criterion_value, self.status()))
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

63
rally/plugins/common/sla/max_average_duration_per_atomic.py
View File

@ -1,4 +1,3 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,61 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.max_average_duration_per_atomic import * # noqa: F401,F403,E501
from rally.plugins.task.sla import max_average_duration_per_atomic as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
import collections
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="max_avg_duration_per_atomic")
class MaxAverageDurationPerAtomic(sla.SLA):
"""Maximum average duration of one iterations atomic actions in seconds."""
CONFIG_SCHEMA = {"type": "object", "$schema": consts.JSON_SCHEMA,
"patternProperties": {".*": {
"type": "number",
"description": "The name of atomic action."}},
"minProperties": 1,
"additionalProperties": False}
def __init__(self, criterion_value):
super(MaxAverageDurationPerAtomic, self).__init__(criterion_value)
self.avg_by_action = collections.defaultdict(float)
self.avg_comp_by_action = collections.defaultdict(
streaming_algorithms.MeanComputation)
self.criterion_items = self.criterion_value.items()
def add_iteration(self, iteration):
if not iteration.get("error"):
for action in iteration["atomic_actions"]:
duration = action["finished_at"] - action["started_at"]
self.avg_comp_by_action[action["name"]].add(duration)
result = self.avg_comp_by_action[action["name"]].result()
self.avg_by_action[action["name"]] = result
self.success = all(self.avg_by_action[atom] <= val
for atom, val in self.criterion_items)
return self.success
def merge(self, other):
for atom, comp in self.avg_comp_by_action.items():
if atom in other.avg_comp_by_action:
comp.merge(other.avg_comp_by_action[atom])
self.avg_by_action = {a: comp.result() or 0.0
for a, comp in self.avg_comp_by_action.items()}
self.success = all(self.avg_by_action[atom] <= val
for atom, val in self.criterion_items)
return self.success
def details(self):
strs = ["Action: '%s'. %.2fs <= %.2fs" %
(atom, self.avg_by_action[atom], val)
for atom, val in self.criterion_items]
head = "Average duration of one iteration for atomic actions:"
end = "Status: %s" % self.status()
return "\n".join([head] + strs + [end])
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

101
rally/plugins/common/sla/outliers.py
View File

@ -1,4 +1,3 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,99 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.outliers import * # noqa: F401,F403
from rally.plugins.task.sla import outliers as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="outliers")
class Outliers(sla.SLA):
"""Limit the number of outliers (iterations that take too much time).
The outliers are detected automatically using the computation of the mean
and standard deviation (std) of the data.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"max": {"type": "integer", "minimum": 0},
"min_iterations": {"type": "integer", "minimum": 3},
"sigmas": {"type": "number", "minimum": 0.0,
"exclusiveMinimum": 0.0}
},
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(Outliers, self).__init__(criterion_value)
self.max_outliers = self.criterion_value.get("max", 0)
# NOTE(msdubov): Having 3 as default is reasonable (need enough data).
self.min_iterations = self.criterion_value.get("min_iterations", 3)
self.sigmas = self.criterion_value.get("sigmas", 3.0)
self.iterations = 0
self.outliers = 0
self.threshold = None
self.mean_comp = streaming_algorithms.MeanComputation()
self.std_comp = streaming_algorithms.StdDevComputation()
def add_iteration(self, iteration):
# NOTE(ikhudoshyn): This method can not be implemented properly.
# After adding a new iteration, both mean and standard deviation
# may change. Hence threshold will change as well. In this case we
# should again compare durations of all accounted iterations
# to the threshold. Unfortunately we can not do it since
# we do not store durations.
# Implementation provided here only gives rough approximation
# of outliers number.
if not iteration.get("error"):
duration = iteration["duration"]
self.iterations += 1
# NOTE(msdubov): First check if the current iteration is an outlier
if (self.iterations >= self.min_iterations
and self.threshold and duration > self.threshold):
self.outliers += 1
# NOTE(msdubov): Then update the threshold value
self.mean_comp.add(duration)
self.std_comp.add(duration)
if self.iterations >= 2:
mean = self.mean_comp.result()
std = self.std_comp.result()
self.threshold = mean + self.sigmas * std
self.success = self.outliers <= self.max_outliers
return self.success
def merge(self, other):
# NOTE(ikhudoshyn): This method can not be implemented properly.
# After merge, both mean and standard deviation may change.
# Hence threshold will change as well. In this case we
# should again compare durations of all accounted iterations
# to the threshold. Unfortunately we can not do it since
# we do not store durations.
# Implementation provided here only gives rough approximation
# of outliers number.
self.iterations += other.iterations
self.outliers += other.outliers
self.mean_comp.merge(other.mean_comp)
self.std_comp.merge(other.std_comp)
if self.iterations >= 2:
mean = self.mean_comp.result()
std = self.std_comp.result()
self.threshold = mean + self.sigmas * std
self.success = self.outliers <= self.max_outliers
return self.success
def details(self):
return ("Maximum number of outliers %i <= %i - %s" %
(self.outliers, self.max_outliers, self.status()))
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

62
rally/plugins/common/sla/performance_degradation.py
View File

@ -1,4 +1,3 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,60 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.task.sla.performance_degradation import * # noqa: F401,F403
from rally.plugins.task.sla import performance_degradation as _new
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from __future__ import division
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
from rally.utils import strutils
# import it as last item to be sure that we use the right module
from rally.common import logging
@sla.configure(name="performance_degradation")
class PerformanceDegradation(sla.SLA):
"""Calculates performance degradation based on iteration time
This SLA plugin finds minimum and maximum duration of
iterations completed without errors during Rally task execution.
Assuming that minimum duration is 100%, it calculates
performance degradation against maximum duration.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"max_degradation": {
"type": "number",
"minimum": 0.0,
},
},
"required": [
"max_degradation",
],
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(PerformanceDegradation, self).__init__(criterion_value)
self.max_degradation = self.criterion_value["max_degradation"]
self.degradation = streaming_algorithms.DegradationComputation()
def add_iteration(self, iteration):
if not iteration.get("error"):
self.degradation.add(iteration["duration"])
self.success = self.degradation.result() <= self.max_degradation
return self.success
def merge(self, other):
self.degradation.merge(other.degradation)
self.success = self.degradation.result() <= self.max_degradation
return self.success
def details(self):
res = strutils.format_float_to_str(self.degradation.result() or 0.0)
return "Current degradation: %s%% - %s" % (res, self.status())
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

61
rally/plugins/common/types.py
View File

@ -12,60 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from rally.plugins.task.types import * # noqa: F401,F403
from rally.plugins.task import types as _new
import requests
from rally.common.plugin import plugin
from rally import exceptions
from rally.task import types
# import it as last item to be sure that we use the right module
from rally.common import logging
@plugin.configure(name="path_or_url")
class PathOrUrl(types.ResourceType):
"""Check whether file exists or url available."""
def pre_process(self, resource_spec, config):
path = os.path.expanduser(resource_spec)
if os.path.isfile(path):
return path
try:
head = requests.head(path, verify=False, allow_redirects=True)
if head.status_code == 200:
return path
raise exceptions.InvalidScenarioArgument(
"Url %s unavailable (code %s)" % (path, head.status_code))
except Exception as ex:
raise exceptions.InvalidScenarioArgument(
"Url error %s (%s)" % (path, ex))
@plugin.configure(name="file")
class FileType(types.ResourceType):
"""Return content of the file by its path."""
def pre_process(self, resource_spec, config):
with open(os.path.expanduser(resource_spec), "r") as f:
return f.read()
@plugin.configure(name="expand_user_path")
class ExpandUserPath(types.ResourceType):
"""Expands user path."""
def pre_process(self, resource_spec, config):
return os.path.expanduser(resource_spec)
@plugin.configure(name="file_dict")
class FileTypeDict(types.ResourceType):
"""Return the dictionary of items with file path and file content."""
def pre_process(self, resource_spec, config):
file_type_dict = {}
for file_path in resource_spec:
file_path = os.path.expanduser(file_path)
with open(file_path, "r") as f:
file_type_dict[file_path] = f.read()
return file_type_dict
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

453
rally/plugins/common/verification/reporters.py
View File

@ -12,452 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import json
import re
from rally.plugins.verification.reporters import * # noqa: F401,F403
from rally.plugins.verification import reporters as _new
from rally.common.io import junit
from rally import consts
from rally.ui import utils as ui_utils
from rally.verification import reporter
# import it as last item to be sure that we use the right module
from rally.common import logging
SKIP_RE = re.compile(r"Skipped until Bug: ?(?P<bug_number>\d+) is resolved.")
LP_BUG_LINK = "https://launchpad.net/bugs/%s"
TIME_FORMAT = consts.TimeFormat.ISO8601
@reporter.configure("json")
class JSONReporter(reporter.VerificationReporter):
"""Generates verification report in JSON format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: json
{"verifications": {
"verification-uuid-1": {
"status": "finished",
"skipped": 1,
"started_at": "2001-01-01T00:00:00",
"finished_at": "2001-01-01T00:05:00",
"tests_duration": 5,
"run_args": {
"pattern": "set=smoke",
"xfail_list": {"some.test.TestCase.test_xfail":
"Some reason why it is expected."},
"skip_list": {"some.test.TestCase.test_skipped":
"This test was skipped intentionally"},
},
"success": 1,
"expected_failures": 1,
"tests_count": 3,
"failures": 0,
"unexpected_success": 0
},
"verification-uuid-2": {
"status": "finished",
"skipped": 1,
"started_at": "2002-01-01T00:00:00",
"finished_at": "2002-01-01T00:05:00",
"tests_duration": 5,
"run_args": {
"pattern": "set=smoke",
"xfail_list": {"some.test.TestCase.test_xfail":
"Some reason why it is expected."},
"skip_list": {"some.test.TestCase.test_skipped":
"This test was skipped intentionally"},
},
"success": 1,
"expected_failures": 1,
"tests_count": 3,
"failures": 1,
"unexpected_success": 0
}
},
"tests": {
"some.test.TestCase.test_foo[tag1,tag2]": {
"name": "some.test.TestCase.test_foo",
"tags": ["tag1","tag2"],
"by_verification": {
"verification-uuid-1": {
"status": "success",
"duration": "1.111"
},
"verification-uuid-2": {
"status": "success",
"duration": "22.222"
}
}
},
"some.test.TestCase.test_skipped[tag1]": {
"name": "some.test.TestCase.test_skipped",
"tags": ["tag1"],
"by_verification": {
"verification-uuid-1": {
"status": "skipped",
"duration": "0",
"details": "Skipped until Bug: 666 is resolved."
},
"verification-uuid-2": {
"status": "skipped",
"duration": "0",
"details": "Skipped until Bug: 666 is resolved."
}
}
},
"some.test.TestCase.test_xfail": {
"name": "some.test.TestCase.test_xfail",
"tags": [],
"by_verification": {
"verification-uuid-1": {
"status": "xfail",
"duration": "3",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
},
"verification-uuid-2": {
"status": "xfail",
"duration": "3",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
}
}
},
"some.test.TestCase.test_failed": {
"name": "some.test.TestCase.test_failed",
"tags": [],
"by_verification": {
"verification-uuid-2": {
"status": "fail",
"duration": "4",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
}
}
}
}
}
"""
@classmethod
def validate(cls, output_destination):
"""Validate destination of report.
:param output_destination: Destination of report
"""
# nothing to check :)
pass
def _generate(self):
"""Prepare raw report."""
verifications = collections.OrderedDict()
tests = {}
for v in self.verifications:
verifications[v.uuid] = {
"started_at": v.created_at.strftime(TIME_FORMAT),
"finished_at": v.updated_at.strftime(TIME_FORMAT),
"status": v.status,
"run_args": v.run_args,
"tests_count": v.tests_count,
"tests_duration": v.tests_duration,
"skipped": v.skipped,
"success": v.success,
"expected_failures": v.expected_failures,
"unexpected_success": v.unexpected_success,
"failures": v.failures,
}
for test_id, result in v.tests.items():
if test_id not in tests:
# NOTE(ylobankov): It is more convenient to see test ID
# at the first place in the report.
tags = sorted(result.get("tags", []), reverse=True,
key=lambda tag: tag.startswith("id-"))
tests[test_id] = {"tags": tags,
"name": result["name"],
"by_verification": {}}
tests[test_id]["by_verification"][v.uuid] = {
"status": result["status"],
"duration": result["duration"]
}
reason = result.get("reason", "")
if reason:
match = SKIP_RE.match(reason)
if match:
link = LP_BUG_LINK % match.group("bug_number")
reason = re.sub(match.group("bug_number"), link,
reason)
traceback = result.get("traceback", "")
sep = "\n\n" if reason and traceback else ""
d = (reason + sep + traceback.strip()) or None
if d:
tests[test_id]["by_verification"][v.uuid]["details"] = d
return {"verifications": verifications, "tests": tests}
def generate(self):
raw_report = json.dumps(self._generate(), indent=4)
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
@reporter.configure("html")
class HTMLReporter(JSONReporter):
"""Generates verification report in HTML format."""
INCLUDE_LIBS = False
# "T" separator of ISO 8601 is not user-friendly enough.
TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
def generate(self):
report = self._generate()
uuids = report["verifications"].keys()
show_comparison_note = False
for test in report["tests"].values():
# make as much as possible processing here to reduce processing
# at JS side
test["has_details"] = False
for test_info in test["by_verification"].values():
if "details" not in test_info:
test_info["details"] = None
elif not test["has_details"]:
test["has_details"] = True
durations = []
# iter by uuids to store right order for comparison
for uuid in uuids:
if uuid in test["by_verification"]:
durations.append(test["by_verification"][uuid]["duration"])
if float(durations[-1]) < 0.001:
durations[-1] = "0"
# not to display such little duration in the report
test["by_verification"][uuid]["duration"] = ""
if len(durations) > 1 and not (
durations[0] == "0" and durations[-1] == "0"):
# compare result with result of the first verification
diff = float(durations[-1]) - float(durations[0])
result = "%s (" % durations[-1]
if diff >= 0:
result += "+"
result += "%s)" % diff
test["by_verification"][uuid]["duration"] = result
if not show_comparison_note and len(durations) > 2:
# NOTE(andreykurilin): only in case of comparison of more
# than 2 results of the same test we should display a note
# about the comparison strategy
show_comparison_note = True
template = ui_utils.get_template("verification/report.html")
context = {"uuids": list(uuids),
"verifications": report["verifications"],
"tests": report["tests"],
"show_comparison_note": show_comparison_note}
raw_report = template.render(data=json.dumps(context),
include_libs=self.INCLUDE_LIBS)
# in future we will support html_static and will need to save more
# files
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
@reporter.configure("html-static")
class HTMLStaticReporter(HTMLReporter):
"""Generates verification report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True
@reporter.configure("junit-xml")
class JUnitXMLReporter(reporter.VerificationReporter):
"""Generates verification report in JUnit-XML format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: xml
<testsuites>
<!--Report is generated by Rally 0.8.0 at 2002-01-01T00:00:00-->
<testsuite id="verification-uuid-1"
tests="9"
time="1.111"
errors="0"
failures="3"
skipped="0"
timestamp="2001-01-01T00:00:00">
<testcase classname="some.test.TestCase"
name="test_foo"
time="8"
timestamp="2001-01-01T00:01:00" />
<testcase classname="some.test.TestCase"
name="test_skipped"
time="0"
timestamp="2001-01-01T00:02:00">
<skipped>Skipped until Bug: 666 is resolved.</skipped>
</testcase>
<testcase classname="some.test.TestCase"
name="test_xfail"
time="3"
timestamp="2001-01-01T00:03:00">
<!--It is an expected failure due to: something-->
<!--Traceback:
HEEELP-->
</testcase>
<testcase classname="some.test.TestCase"
name="test_uxsuccess"
time="3"
timestamp="2001-01-01T00:04:00">
<failure>
It is an unexpected success. The test should fail due to:
It should fail, I said!
</failure>
</testcase>
</testsuite>
<testsuite id="verification-uuid-2"
tests="99"
time="22.222"
errors="0"
failures="33"
skipped="0"
timestamp="2002-01-01T00:00:00">
<testcase classname="some.test.TestCase"
name="test_foo"
time="8"
timestamp="2001-02-01T00:01:00" />
<testcase classname="some.test.TestCase"
name="test_failed"
time="8"
timestamp="2001-02-01T00:02:00">
<failure>HEEEEEEELP</failure>
</testcase>
<testcase classname="some.test.TestCase"
name="test_skipped"
time="0"
timestamp="2001-02-01T00:03:00">
<skipped>Skipped until Bug: 666 is resolved.</skipped>
</testcase>
<testcase classname="some.test.TestCase"
name="test_xfail"
time="4"
timestamp="2001-02-01T00:04:00">
<!--It is an expected failure due to: something-->
<!--Traceback:
HEEELP-->
</testcase>
</testsuite>
</testsuites>
"""
@classmethod
def validate(cls, output_destination):
pass
def generate(self):
report = junit.JUnitXML()
for v in self.verifications:
test_suite = report.add_test_suite(
id=v.uuid,
time=str(v.tests_duration),
timestamp=v.created_at.strftime(TIME_FORMAT)
)
test_suite.setup_final_stats(
tests=str(v.tests_count),
skipped=str(v.skipped),
failures=str(v.failures + v.unexpected_success)
)
tests = sorted(v.tests.values(),
key=lambda t: (t.get("timestamp", ""), t["name"]))
for result in tests:
class_name, name = result["name"].rsplit(".", 1)
test_id = [tag[3:] for tag in result.get("tags", [])
if tag.startswith("id-")]
test_case = test_suite.add_test_case(
id=(test_id[0] if test_id else None),
time=result["duration"], name=name, classname=class_name,
timestamp=result.get("timestamp"))
if result["status"] == "success":
# nothing to add
pass
elif result["status"] == "uxsuccess":
test_case.mark_as_uxsuccess(
result.get("reason"))
elif result["status"] == "fail":
test_case.mark_as_failed(
result.get("traceback", None))
elif result["status"] == "xfail":
trace = result.get("traceback", None)
test_case.mark_as_xfail(
result.get("reason", None),
f"Traceback:\n{trace}" if trace else None)
elif result["status"] == "skip":
test_case.mark_as_skipped(
result.get("reason", None))
else:
# wtf is it?! we should add validation of results...
pass
raw_report = report.to_string()
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

149
rally/plugins/common/verification/testr.py
View File

@ -12,150 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import shutil
import subprocess
from rally.plugins.verification.testr import * # noqa: F401,F403
from rally.plugins.verification import testr as _new
from rally.common.io import subunit_v2
# import it as last item to be sure that we use the right module
from rally.common import logging
from rally.common import utils as common_utils
from rally import exceptions
from rally.verification import context
from rally.verification import manager
from rally.verification import utils
LOG = logging.getLogger(__name__)
TEST_NAME_RE = re.compile(r"^[a-zA-Z_.0-9]+(\[[a-zA-Z-_,=0-9]*\])?$")
@context.configure("testr", order=999)
class TestrContext(context.VerifierContext):
"""Context to transform 'run_args' into CLI arguments for testr."""
def __init__(self, ctx):
super(TestrContext, self).__init__(ctx)
self._tmp_files = []
def setup(self):
super(TestrContext, self).setup()
use_testr = getattr(self.verifier.manager, "_use_testr", True)
if use_testr:
base_cmd = "testr"
else:
base_cmd = "stestr"
self.context["testr_cmd"] = [base_cmd, "run", "--subunit"]
run_args = self.verifier.manager.prepare_run_args(
self.context.get("run_args", {}))
concurrency = run_args.get("concurrency", 0)
if concurrency == 0 or concurrency > 1:
if use_testr:
self.context["testr_cmd"].append("--parallel")
if concurrency >= 1:
if concurrency == 1 and not use_testr:
self.context["testr_cmd"].append("--serial")
else:
self.context["testr_cmd"].extend(
["--concurrency", str(concurrency)])
load_list = self.context.get("load_list")
skip_list = self.context.get("skip_list")
if skip_list:
load_list = set(load_list) - set(skip_list)
if load_list:
load_list_file = common_utils.generate_random_path()
with open(load_list_file, "w") as f:
f.write("\n".join(load_list))
self._tmp_files.append(load_list_file)
self.context["testr_cmd"].extend(["--load-list", load_list_file])
if run_args.get("failed"):
self.context["testr_cmd"].append("--failing")
if run_args.get("pattern"):
self.context["testr_cmd"].append(run_args.get("pattern"))
def cleanup(self):
for f in self._tmp_files:
if os.path.exists(f):
os.remove(f)
class TestrLauncher(manager.VerifierManager):
"""Testr/sTestr wrapper."""
def __init__(self, *args, **kwargs):
super(TestrLauncher, self).__init__(*args, **kwargs)
self._use_testr = os.path.exists(os.path.join(
self.repo_dir, ".testr.conf"))
@property
def run_environ(self):
return self.environ
def _init_testr(self):
"""Initialize testr."""
test_repository_dir = os.path.join(self.base_dir, ".testrepository")
# NOTE(andreykurilin): Is there any possibility that .testrepository
# presents in clear repo?!
if not os.path.isdir(test_repository_dir):
LOG.debug("Initializing testr.")
if self._use_testr:
base_cmd = "testr"
else:
base_cmd = "stestr"
try:
utils.check_output([base_cmd, "init"], cwd=self.repo_dir,
env=self.environ)
except (subprocess.CalledProcessError, OSError):
if os.path.exists(test_repository_dir):
shutil.rmtree(test_repository_dir)
raise exceptions.RallyException("Failed to initialize testr.")
def install(self):
super(TestrLauncher, self).install()
self._init_testr()
def list_tests(self, pattern=""):
"""List all tests."""
if self._use_testr:
cmd = ["testr", "list-tests", pattern]
else:
cmd = ["stestr", "list", pattern]
output = utils.check_output(cmd,
cwd=self.repo_dir, env=self.environ,
debug_output=False)
return [t for t in output.split("\n") if TEST_NAME_RE.match(t)]
def run(self, context):
"""Run tests."""
testr_cmd = context["testr_cmd"]
LOG.debug("Test(s) started by the command: '%s'."
% " ".join(testr_cmd))
stream = subprocess.Popen(testr_cmd, env=self.run_environ,
cwd=self.repo_dir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
xfail_list = context.get("xfail_list")
skip_list = context.get("skip_list")
results = subunit_v2.parse(stream.stdout, live=True,
expected_failures=xfail_list,
skipped_tests=skip_list,
logger_name=self.verifier.name)
stream.wait()
return results
def prepare_run_args(self, run_args):
"""Prepare 'run_args' for testr context.
This method is called by TestrContext before transforming 'run_args'
into CLI arguments for testr.
"""
return run_args
logging.log_deprecated_module(
target=__name__, new_module=_new.__name__, release="3.0.0"
)

0
rally/plugins/common/contexts/__init__.py → rally/plugins/task/contexts/__init__.py
View File

0
rally/plugins/common/contexts/dummy.py → rally/plugins/task/contexts/dummy.py
View File

0
rally/plugins/common/scenarios/dummy/__init__.py → rally/plugins/task/exporters/__init__.py
View File

0
tests/unit/plugins/common/contexts/__init__.py → rally/plugins/task/exporters/elastic/__init__.py
View File

159
rally/plugins/task/exporters/elastic/client.py Executable file
View File

@ -0,0 +1,159 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import requests
from rally.common import logging
from rally import exceptions
LOG = logging.getLogger(__name__)
class ElasticSearchClient(object):
"""The helper class for communication with ElasticSearch 2.*, 5.*, 6.*"""
# a number of documents to push to the cluster at once.
CHUNK_LENGTH = 10000
def __init__(self, url):
self._url = url.rstrip("/") if url else "http://localhost:9200"
self._version = None
@staticmethod
def _check_response(resp, action=None):
if resp.status_code in (200, 201):
return
# it is an error. let's try to find the reason
reason = None
try:
data = resp.json()
except ValueError:
# it is ok
pass
else:
if "error" in data:
if isinstance(data["error"], dict):
reason = data["error"].get("reason", "")
else:
reason = data["error"]
reason = reason or resp.text or "n/a"
action = action or "connect to"
raise exceptions.RallyException(
"[HTTP %s] Failed to %s ElasticSearch cluster: %s" %
(resp.status_code, action, reason))
def version(self):
"""Get version of the ElasticSearch cluster."""
if self._version is None:
self.info()
return self._version
def info(self):
"""Retrieve info about the ElasticSearch cluster."""
resp = requests.get(self._url)
self._check_response(resp)
err_msg = "Failed to retrieve info about the ElasticSearch cluster: %s"
try:
data = resp.json()
except ValueError:
LOG.debug("Return data from %s: %s" % (self._url, resp.text))
raise exceptions.RallyException(
err_msg % "The return data doesn't look like a json.")
version = data.get("version", {}).get("number")
if not version:
LOG.debug("Return data from %s: %s" % (self._url, resp.text))
raise exceptions.RallyException(
err_msg % "Failed to parse the received data.")
self._version = version
if self._version.startswith("2"):
data["version"]["build_date"] = data["version"].pop(
"build_timestamp")
return data
def push_documents(self, documents):
"""Push documents to the ElasticSearch cluster using bulk API.
:param documents: a list of documents to push
"""
LOG.debug("Pushing %s documents by chunks (up to %s documents at once)"
" to ElasticSearch." %
# dividing numbers by two, since each documents has 2 lines
# in `documents` (action and document itself).
(len(documents) / 2, self.CHUNK_LENGTH / 2))
for pos in range(0, len(documents), self.CHUNK_LENGTH):
data = "\n".join(documents[pos:pos + self.CHUNK_LENGTH]) + "\n"
raw_resp = requests.post(
self._url + "/_bulk", data=data,
headers={"Content-Type": "application/x-ndjson"}
)
self._check_response(raw_resp, action="push documents to")
LOG.debug("Successfully pushed %s documents." %
len(raw_resp.json()["items"]))
def list_indices(self):
"""List all indices."""
resp = requests.get(self._url + "/_cat/indices?v")
self._check_response(resp, "list the indices at")
return resp.text.rstrip().split(" ")
def create_index(self, name, doc_type, properties):
"""Create an index.
There are two very different ways to search strings. You can either
search whole values, that we often refer to as keyword search, or
individual tokens, that we usually refer to as full-text search.
In ElasticSearch 2.x `string` data type is used for these cases whereas
ElasticSearch 5.0 the `string` data type was replaced by two new types:
`keyword` and `text`. Since it is hard to predict the destiny of
`string` data type and support of 2 formats of input data, the
properties should be transmitted in ElasticSearch 5.x format.
"""
if self.version().startswith("2."):
properties = copy.deepcopy(properties)
for spec in properties.values():
if spec.get("type", None) == "text":
spec["type"] = "string"
elif spec.get("type", None) == "keyword":
spec["type"] = "string"
spec["index"] = "not_analyzed"
resp = requests.put(
self._url + "/%s" % name,
json={"mappings": {doc_type: {"properties": properties}}})
self._check_response(resp, "create index at")
def check_document(self, index, doc_id, doc_type="data"):
"""Check for the existence of a document.
:param index: The index of a document
:param doc_id: The ID of a document
:param doc_type: The type of a document (Defaults to data)
"""
resp = requests.head("%(url)s/%(index)s/%(type)s/%(id)s" %
{"url": self._url,
"index": index,
"type": doc_type,
"id": doc_id})
if resp.status_code == 200:
return True
elif resp.status_code == 404:
return False
else:
self._check_response(resp, "check the index at")

386
rally/plugins/task/exporters/elastic/exporter.py Executable file
View File

@ -0,0 +1,386 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import datetime as dt
import itertools
import json
import os
from rally.common import logging
from rally.common import validation
from rally import consts
from rally import exceptions
from rally.plugins.task.exporters.elastic import client
from rally.plugins.task.exporters.elastic import flatten
from rally.task import exporter
LOG = logging.getLogger(__name__)
@validation.configure("es_exporter_destination")
class Validator(validation.Validator):
"""Validates the destination for ElasticSearch exporter.
In case when the destination is ElasticSearch cluster, the version of it
should be 2.* or 5.*
"""
def validate(self, context, config, plugin_cls, plugin_cfg):
destination = plugin_cfg["destination"]
if destination and (not destination.startswith("http://")
and not destination.startswith("https://")):
# it is a path to a local file
return
es = client.ElasticSearchClient(destination)
try:
version = es.version()
except exceptions.RallyException as e:
# re-raise a proper exception to hide redundant traceback
self.fail(e.format_message())
if not (version.startswith("2.")
or version.startswith("5.")
or version.startswith("6.")):
self.fail("The unsupported version detected %s." % version)
@validation.add("es_exporter_destination")
@exporter.configure("elastic")
class ElasticSearchExporter(exporter.TaskExporter):
"""Exports task results to the ElasticSearch 2.x, 5.x or 6.x clusters.
The exported data includes:
* Task basic information such as title, description, status,
deployment uuid, etc.
See rally_task_v1_data index.
* Workload information such as scenario name and configuration, runner
type and configuration, time of the start load, success rate, sla
details in case of errors, etc.
See rally_workload_v1_data index.
* Separate documents for all atomic actions.
See rally_atomic_action_data_v1 index.
The destination can be a remote server. In this case specify it like:
https://elastic:changeme@example.com
Or we can dump documents to the file. The destination should look like:
/home/foo/bar.txt
In case of an empty destination, the http://localhost:9200 destination
will be used.
"""
TASK_INDEX = "rally_task_data_v1"
WORKLOAD_INDEX = "rally_workload_data_v1"
AA_INDEX = "rally_atomic_action_data_v1"
INDEX_SCHEMAS = {
TASK_INDEX: {
"task_uuid": {"type": "keyword"},
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"title": {"type": "text"},
"description": {"type": "text"},
"status": {"type": "keyword"},
"pass_sla": {"type": "boolean"},
"tags": {"type": "keyword"}
},
WORKLOAD_INDEX: {
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"scenario_name": {"type": "keyword"},
"scenario_cfg": {"type": "keyword"},
"description": {"type": "text"},
"runner_name": {"type": "keyword"},
"runner_cfg": {"type": "keyword"},
"contexts": {"type": "keyword"},
"task_uuid": {"type": "keyword"},
"subtask_uuid": {"type": "keyword"},
"started_at": {"type": "date"},
"load_duration": {"type": "long"},
"full_duration": {"type": "long"},
"pass_sla": {"type": "boolean"},
"success_rate": {"type": "float"},
"sla_details": {"type": "text"}
},
AA_INDEX: {
"deployment_uuid": {"type": "keyword"},
"deployment_name": {"type": "keyword"},
"action_name": {"type": "keyword"},
"workload_uuid": {"type": "keyword"},
"scenario_cfg": {"type": "keyword"},
"contexts": {"type": "keyword"},
"runner_name": {"type": "keyword"},
"runner_cfg": {"type": "keyword"},
"success": {"type": "boolean"},
"duration": {"type": "float"},
"started_at": {"type": "date"},
"finished_at": {"type": "date"},
"parent": {"type": "keyword"},
"error": {"type": "keyword"}
}
}
def __init__(self, tasks_results, output_destination, api=None):
super(ElasticSearchExporter, self).__init__(tasks_results,
output_destination,
api=api)
self._report = []
self._remote = (
output_destination is None or (
output_destination.startswith("http://")
or self.output_destination.startswith("https://")))
if self._remote:
self._client = client.ElasticSearchClient(self.output_destination)
def _add_index(self, index, body, doc_id=None, doc_type="data"):
"""Create a document for the specified index with specified id.
:param index: The name of the index
:param body: The document. Here is the report of (sla,
scenario, iteration and atomic action)
:param doc_id: Document ID. Here we use task/subtask/workload uuid
:param doc_type: The type of document
"""
self._report.append(
json.dumps(
# use OrderedDict to make the report more unified
{"index": collections.OrderedDict([
("_index", index),
("_type", doc_type),
("_id", doc_id)])},
sort_keys=False))
self._report.append(json.dumps(body))
def _ensure_indices(self):
"""Check available indices and create require ones if they missed."""
available_index = set(self._client.list_indices())
missed_index = {self.TASK_INDEX, self.WORKLOAD_INDEX,
self.AA_INDEX} - available_index
for index in missed_index:
LOG.debug("Creating '%s' index." % index)
self._client.create_index(index, doc_type="data",
properties=self.INDEX_SCHEMAS[index])
@staticmethod
def _make_action_report(name, workload_id, workload, duration,
started_at, finished_at, parent, error):
# NOTE(andreykurilin): actually, this method just creates a dict object
# but we need to have the same format at two places, so the template
# transformed into a method.
parent = parent[0] if parent else None
return {
"deployment_uuid": workload["deployment_uuid"],
"deployment_name": workload["deployment_name"],
"action_name": name,
"workload_uuid": workload_id,
"scenario_cfg": workload["scenario_cfg"],
"contexts": workload["contexts"],
"runner_name": workload["runner_name"],
"runner_cfg": workload["runner_cfg"],
"success": not bool(error),
"duration": duration,
"started_at": started_at,
"finished_at": finished_at,
"parent": parent,
"error": error
}
def _process_atomic_actions(self, itr, workload, workload_id,
atomic_actions=None, _parent=None, _depth=0,
_cache=None):
"""Process atomic actions of an iteration
:param atomic_actions: A list with an atomic actions
:param itr: The iteration data
:param workload: The workload report
:param workload_id: The workload UUID
:param _parent: An inner parameter which is used for pointing to the
parent atomic action
:param _depth: An inner parameter which is used to mark the level of
depth while parsing atomic action children
:param _cache: An inner parameter which is used to avoid conflicts in
IDs of atomic actions of a single iteration.
"""
if _depth >= 3:
return
cache = _cache or {}
if atomic_actions is None:
atomic_actions = itr["atomic_actions"]
act_id_tmpl = "%(itr_id)s_action_%(action_name)s_%(num)s"
for i, action in enumerate(atomic_actions, 1):
cache.setdefault(action["name"], 0)
act_id = act_id_tmpl % {
"itr_id": itr["id"],
"action_name": action["name"],
"num": cache[action["name"]]}
cache[action["name"]] += 1
started_at = dt.datetime.utcfromtimestamp(action["started_at"])
finished_at = dt.datetime.utcfromtimestamp(action["finished_at"])
started_at = started_at.strftime(consts.TimeFormat.ISO8601)
finished_at = finished_at.strftime(consts.TimeFormat.ISO8601)
action_report = self._make_action_report(
name=action["name"],
workload_id=workload_id,
workload=workload,
duration=(action["finished_at"] - action["started_at"]),
started_at=started_at,
finished_at=finished_at,
parent=_parent,
error=(itr["error"] if action.get("failed", False) else None)
)
self._add_index(self.AA_INDEX, action_report,
doc_id=act_id)
self._process_atomic_actions(
atomic_actions=action["children"],
itr=itr,
workload=workload,
workload_id=workload_id,
_parent=(act_id, action_report),
_depth=(_depth + 1),
_cache=cache)
if itr["error"] and (
# the case when it is a top level of the scenario and the
# first fails the item which is not wrapped by AtomicTimer
(not _parent and not atomic_actions)
# the case when it is a top level of the scenario and and
# the item fails after some atomic actions completed
or (not _parent and atomic_actions
and not atomic_actions[-1].get("failed", False))):
act_id = act_id_tmpl % {
"itr_id": itr["id"],
"action_name": "no-name-action",
"num": 0}
# Since the action had not be wrapped by AtomicTimer, we cannot
# make any assumption about it's duration (start_time) so let's use
# finished_at timestamp of iteration with 0 duration
timestamp = (itr["timestamp"] + itr["duration"]
+ itr["idle_duration"])
timestamp = dt.datetime.utcfromtimestamp(timestamp)
timestamp = timestamp.strftime(consts.TimeFormat.ISO8601)
action_report = self._make_action_report(
name="no-name-action",
workload_id=workload_id,
workload=workload,
duration=0,
started_at=timestamp,
finished_at=timestamp,
parent=_parent,
error=itr["error"]
)
self._add_index(self.AA_INDEX, action_report, doc_id=act_id)
def generate(self):
if self._remote:
self._ensure_indices()
for task in self.tasks_results:
if self._remote:
if self._client.check_document(self.TASK_INDEX, task["uuid"]):
raise exceptions.RallyException(
"Failed to push the task %s to the ElasticSearch "
"cluster. The document with such UUID already exists" %
task["uuid"])
task_report = {
"task_uuid": task["uuid"],
"deployment_uuid": task["env_uuid"],
"deployment_name": task["env_name"],
"title": task["title"],
"description": task["description"],
"status": task["status"],
"pass_sla": task["pass_sla"],
"tags": task["tags"]
}
self._add_index(self.TASK_INDEX, task_report,
doc_id=task["uuid"])
# NOTE(andreykurilin): The subtasks do not have much logic now, so
# there is no reason to save the info about them.
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
durations = workload["statistics"]["durations"]
success_rate = durations["total"]["data"]["success"]
if success_rate == "n/a":
success_rate = 0.0
else:
# cut the % char and transform to the float value
success_rate = float(success_rate[:-1]) / 100.0
started_at = workload["start_time"]
if started_at:
started_at = dt.datetime.utcfromtimestamp(started_at)
started_at = started_at.strftime(consts.TimeFormat.ISO8601)
workload_report = {
"task_uuid": workload["task_uuid"],
"subtask_uuid": workload["subtask_uuid"],
"deployment_uuid": task["env_uuid"],
"deployment_name": task["env_name"],
"scenario_name": workload["name"],
"scenario_cfg": flatten.transform(workload["args"]),
"description": workload["description"],
"runner_name": workload["runner_type"],
"runner_cfg": flatten.transform(workload["runner"]),
"contexts": flatten.transform(workload["contexts"]),
"started_at": started_at,
"load_duration": workload["load_duration"],
"full_duration": workload["full_duration"],
"pass_sla": workload["pass_sla"],
"success_rate": success_rate,
"sla_details": [s["detail"]
for s in workload["sla_results"]["sla"]
if not s["success"]]}
# do we need to store hooks ?!
self._add_index(self.WORKLOAD_INDEX, workload_report,
doc_id=workload["uuid"])
# Iterations
for idx, itr in enumerate(workload.get("data", []), 1):
itr["id"] = "%(uuid)s_iter_%(num)s" % {
"uuid": workload["uuid"],
"num": str(idx)}
self._process_atomic_actions(
itr=itr,
workload=workload_report,
workload_id=workload["uuid"])
if self._remote:
LOG.debug("The info of ElasticSearch cluster to which the results "
"will be exported: %s" % self._client.info())
self._client.push_documents(self._report)
msg = ("Successfully exported results to ElasticSearch at url "
"'%s'" % self.output_destination)
return {"print": msg}
else:
# a new line is required in the end of the file.
report = "\n".join(self._report) + "\n"
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}

65
rally/plugins/task/exporters/elastic/flatten.py Normal file
View File

@ -0,0 +1,65 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def _join_keys(first, second):
if not second:
return first
elif second.startswith("["):
return "%s%s" % (first, second)
else:
return "%s.%s" % (first, second)
def _process(obj):
if isinstance(obj, (str, bytes)):
yield "", obj
elif isinstance(obj, dict):
for first, tmp_value in obj.items():
for second, value in _process(tmp_value):
yield _join_keys(first, second), value
elif isinstance(obj, (list, tuple)):
for i, tmp_value in enumerate(obj):
for second, value in _process(tmp_value):
yield _join_keys("[%s]" % i, second), value
else:
try:
yield "", "%s" % obj
except Exception:
raise ValueError("Cannot transform obj of '%s' type to flatten "
"structure." % type(obj))
def transform(obj):
"""Transform object to a flatten structure.
Example:
IN:
{"foo": ["xxx", "yyy", {"bar": {"zzz": ["Hello", "World!"]}}]}
OUTPUT:
[
"foo[0]=xxx",
"foo[1]=yyy",
"foo[2].bar.zzz[0]=Hello",
"foo[2].bar.zzz[1]=World!"
]
"""
result = []
for key, value in _process(obj):
if key:
result.append("%s=%s" % (key, value))
else:
result.append(value)
return sorted(result)

56
rally/plugins/task/exporters/html.py Normal file
View File

@ -0,0 +1,56 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import os
from rally.task import exporter
from rally.task.processing import plot
@exporter.configure("html")
class HTMLExporter(exporter.TaskExporter):
"""Generates task report in HTML format."""
INCLUDE_LIBS = False
def _generate_results(self):
results = []
processed_names = {}
for task in self.tasks_results:
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
if workload["name"] in processed_names:
processed_names[workload["name"]] += 1
workload["position"] = processed_names[workload["name"]]
else:
processed_names[workload["name"]] = 0
results.append(task)
return results
def generate(self):
report = plot.plot(self._generate_results(),
include_libs=self.INCLUDE_LIBS)
if self.output_destination:
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": report}
@exporter.configure("html-static")
class HTMLStaticExporter(HTMLExporter):
"""Generates task report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True

123
rally/plugins/task/exporters/json_exporter.py Normal file
View File

@ -0,0 +1,123 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import datetime as dt
import json
from rally.common import version as rally_version
from rally.task import exporter
TIMEFORMAT = "%Y-%m-%dT%H:%M:%S"
@exporter.configure("json")
class JSONExporter(exporter.TaskExporter):
"""Generates task report in JSON format."""
# Revisions:
# 1.0 - the json report v1
# 1.1 - add `contexts_results` key with contexts execution results of
# workloads.
# 1.2 - add `env_uuid` and `env_uuid` which represent environment name
# and UUID where task was executed
REVISION = "1.2"
def _generate_tasks(self):
tasks = []
for task in self.tasks_results:
subtasks = []
for subtask in task["subtasks"]:
workloads = []
for workload in subtask["workloads"]:
hooks = [{
"config": {"action": dict([h["config"]["action"]]),
"trigger": dict([h["config"]["trigger"]]),
"description": h["config"]["description"]},
"results": h["results"],
"summary": h["summary"], } for h in workload["hooks"]]
workloads.append(
collections.OrderedDict(
[("uuid", workload["uuid"]),
("description", workload["description"]),
("runner", {
workload["runner_type"]: workload["runner"]}),
("hooks", hooks),
("scenario", {
workload["name"]: workload["args"]}),
("min_duration", workload["min_duration"]),
("max_duration", workload["max_duration"]),
("start_time", workload["start_time"]),
("load_duration", workload["load_duration"]),
("full_duration", workload["full_duration"]),
("statistics", workload["statistics"]),
("data", workload["data"]),
("failed_iteration_count",
workload["failed_iteration_count"]),
("total_iteration_count",
workload["total_iteration_count"]),
("created_at", workload["created_at"]),
("updated_at", workload["updated_at"]),
("contexts", workload["contexts"]),
("contexts_results",
workload["contexts_results"]),
("position", workload["position"]),
("pass_sla", workload["pass_sla"]),
("sla_results", workload["sla_results"]),
("sla", workload["sla"])]
)
)
subtasks.append(
collections.OrderedDict(
[("uuid", subtask["uuid"]),
("title", subtask["title"]),
("description", subtask["description"]),
("status", subtask["status"]),
("created_at", subtask["created_at"]),
("updated_at", subtask["updated_at"]),
("sla", subtask["sla"]),
("workloads", workloads)]
)
)
tasks.append(
collections.OrderedDict(
[("uuid", task["uuid"]),
("title", task["title"]),
("description", task["description"]),
("status", task["status"]),
("tags", task["tags"]),
("env_uuid", task.get("env_uuid", "n\a")),
("env_name", task.get("env_name", "n\a")),
("created_at", task["created_at"]),
("updated_at", task["updated_at"]),
("pass_sla", task["pass_sla"]),
("subtasks", subtasks)]
)
)
return tasks
def generate(self):
results = {"info": {"rally_version": rally_version.version_string(),
"generated_at": dt.datetime.strftime(
dt.datetime.utcnow(), TIMEFORMAT),
"format_version": self.REVISION},
"tasks": self._generate_tasks()}
results = json.dumps(results, sort_keys=False, indent=4)
if self.output_destination:
return {"files": {self.output_destination: results},
"open": "file://" + self.output_destination}
else:
return {"print": results}

96
rally/plugins/task/exporters/junit.py Normal file
View File

@ -0,0 +1,96 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime as dt
import itertools
import os
from rally.common.io import junit
from rally.task import exporter
@exporter.configure("junit-xml")
class JUnitXMLExporter(exporter.TaskExporter):
"""Generates task report in JUnit-XML format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: xml
<testsuites>
<!--Report is generated by Rally 0.10.0 at 2017-06-04T05:14:00-->
<testsuite id="task-uu-ii-dd"
errors="0"
failures="1"
skipped="0"
tests="2"
time="75.0"
timestamp="2017-06-04T05:14:00">
<testcase classname="CinderVolumes"
name="list_volumes"
id="workload-1-uuid"
time="29.9695231915"
timestamp="2017-06-04T05:14:44" />
<testcase classname="NovaServers"
name="list_keypairs"
id="workload-2-uuid"
time="5"
timestamp="2017-06-04T05:15:15">
<failure>ooops</failure>
</testcase>
</testsuite>
</testsuites>
"""
def generate(self):
root = junit.JUnitXML()
for t in self.tasks_results:
created_at = dt.datetime.strptime(t["created_at"],
"%Y-%m-%dT%H:%M:%S")
updated_at = dt.datetime.strptime(t["updated_at"],
"%Y-%m-%dT%H:%M:%S")
test_suite = root.add_test_suite(
id=t["uuid"],
time="%.2f" % (updated_at - created_at).total_seconds(),
timestamp=t["created_at"]
)
for workload in itertools.chain(
*[s["workloads"] for s in t["subtasks"]]):
class_name, name = workload["name"].split(".", 1)
test_case = test_suite.add_test_case(
id=workload["uuid"],
time="%.2f" % workload["full_duration"],
classname=class_name,
name=name,
timestamp=workload["created_at"]
)
if not workload["pass_sla"]:
details = "\n".join(
[s["detail"]
for s in workload["sla_results"]["sla"]
if not s["success"]]
)
test_case.mark_as_failed(details)
raw_report = root.to_string()
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": raw_report}

40
rally/plugins/task/exporters/trends.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright 2018: ZTE Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from rally.task import exporter
from rally.task.processing import plot
@exporter.configure("trends-html")
class TrendsExporter(exporter.TaskExporter):
"""Generates task trends report in HTML format."""
INCLUDE_LIBS = False
def generate(self):
report = plot.trends(self.tasks_results, self.INCLUDE_LIBS)
if self.output_destination:
return {"files": {self.output_destination: report},
"open": "file://" + os.path.abspath(
self.output_destination)}
else:
return {"print": report}
@exporter.configure("trends-html-static")
class TrendsStaticExport(TrendsExporter):
"""Generates task trends report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True

0
tests/unit/plugins/common/exporters/__init__.py → rally/plugins/task/hook_triggers/__init__.py
View File

74
rally/plugins/task/hook_triggers/event.py Normal file
View File

@ -0,0 +1,74 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally import consts
from rally.task import hook
@hook.configure(name="event")
class EventTrigger(hook.HookTrigger):
"""Triggers hook on specified event and list of values."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"oneOf": [
{
"description": "Triage hook based on specified seconds after "
"start of workload.",
"properties": {
"unit": {"enum": ["time"]},
"at": {
"type": "array",
"minItems": 1,
"uniqueItems": True,
"items": {
"type": "integer",
"minimum": 0
}
},
},
"required": ["unit", "at"],
"additionalProperties": False,
},
{
"description": "Triage hook based on specific iterations.",
"properties": {
"unit": {"enum": ["iteration"]},
"at": {
"type": "array",
"minItems": 1,
"uniqueItems": True,
"items": {
"type": "integer",
"minimum": 1,
}
},
},
"required": ["unit", "at"],
"additionalProperties": False,
},
]
}
def get_listening_event(self):
return self.config["unit"]
def on_event(self, event_type, value=None):
if not (event_type == self.get_listening_event()
and value in self.config["at"]):
# do nothing
return
super(EventTrigger, self).on_event(event_type, value)

69
rally/plugins/task/hook_triggers/periodic.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally import consts
from rally.task import hook
@hook.configure(name="periodic")
class PeriodicTrigger(hook.HookTrigger):
"""Periodically triggers hook with specified range and step."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"oneOf": [
{
"description": "Periodically triage hook based on elapsed time"
" after start of workload.",
"properties": {
"unit": {"enum": ["time"]},
"start": {"type": "integer", "minimum": 0},
"end": {"type": "integer", "minimum": 1},
"step": {"type": "integer", "minimum": 1},
},
"required": ["unit", "step"],
"additionalProperties": False,
},
{
"description": "Periodically triage hook based on iterations.",
"properties": {
"unit": {"enum": ["iteration"]},
"start": {"type": "integer", "minimum": 1},
"end": {"type": "integer", "minimum": 1},
"step": {"type": "integer", "minimum": 1},
},
"required": ["unit", "step"],
"additionalProperties": False,
},
]
}
def __init__(self, context, task, hook_cls):
super(PeriodicTrigger, self).__init__(context, task, hook_cls)
self.config.setdefault(
"start", 0 if self.config["unit"] == "time" else 1)
self.config.setdefault("end", float("Inf"))
def get_listening_event(self):
return self.config["unit"]
def on_event(self, event_type, value=None):
if not (event_type == self.get_listening_event()
and self.config["start"] <= value <= self.config["end"]
and (value - self.config["start"]) % self.config["step"] == 0):
# do nothing
return
super(PeriodicTrigger, self).on_event(event_type, value)

0
tests/unit/plugins/common/exporters/elastic/__init__.py → rally/plugins/task/hooks/__init__.py
View File

68
rally/plugins/task/hooks/sys_call.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import shlex
import subprocess
from rally.common import logging
from rally import consts
from rally import exceptions
from rally.task import hook
LOG = logging.getLogger(__name__)
@hook.configure(name="sys_call")
class SysCallHook(hook.HookAction):
"""Performs system call."""
CONFIG_SCHEMA = {
"$schema": consts.JSON_SCHEMA,
"type": "string",
"description": "Command to execute."
}
def run(self):
LOG.debug("sys_call hook: Running command %s" % self.config)
proc = subprocess.Popen(shlex.split(self.config),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
out, err = proc.communicate()
LOG.debug("sys_call hook: Command %s returned %s"
% (self.config, proc.returncode))
if proc.returncode:
self.set_error(
exception_name="n/a", # no exception class
description="Subprocess returned %s" % proc.returncode,
details=(err or "stdout: %s" % out))
# NOTE(amaretskiy): Try to load JSON for charts,
# otherwise save output as-is
try:
output = json.loads(out)
for arg in ("additive", "complete"):
for out_ in output.get(arg, []):
self.add_output(**{arg: out_})
except (TypeError, ValueError, exceptions.RallyException):
self.add_output(
complete={"title": "System call",
"chart_plugin": "TextArea",
"description": "Args: %s" % self.config,
"data": ["RetCode: %i" % proc.returncode,
"StdOut: %s" % (out or "(empty)"),
"StdErr: %s" % (err or "(empty)")]})

0
tests/unit/plugins/common/hook/__init__.py → rally/plugins/task/runners/__init__.py
View File

342
rally/plugins/task/runners/constant.py Normal file
View File

@ -0,0 +1,342 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import multiprocessing
import queue as Queue
import threading
import time
from rally.common import utils
from rally.common import validation
from rally import consts
from rally.task import runner
def _worker_process(queue, iteration_gen, timeout, concurrency, times,
duration, context, cls, method_name, args, event_queue,
aborted, info):
"""Start the scenario within threads.
Spawn threads to support scenario execution.
Scenario is ran for a fixed number of times if times is specified
Scenario is ran for fixed duration if duration is specified.
This generates a constant load on the cloud under test by executing each
scenario iteration without pausing between iterations. Each thread runs
the scenario method once with passed scenario arguments and context.
After execution the result is appended to the queue.
:param queue: queue object to append results
:param iteration_gen: next iteration number generator
:param timeout: operation's timeout
:param concurrency: number of concurrently running scenario iterations
:param times: total number of scenario iterations to be run
:param duration: total duration in seconds of the run
:param context: scenario context object
:param cls: scenario class
:param method_name: scenario method name
:param args: scenario args
:param event_queue: queue object to append events
:param aborted: multiprocessing.Event that aborts load generation if
the flag is set
:param info: info about all processes count and counter of launched process
"""
def _to_be_continued(iteration, current_duration, aborted, times=None,
duration=None):
if times is not None:
return iteration < times and not aborted.is_set()
elif duration is not None:
return current_duration < duration and not aborted.is_set()
else:
return False
if times is None and duration is None:
raise ValueError("times or duration must be specified")
pool = collections.deque()
alive_threads_in_pool = 0
finished_threads_in_pool = 0
runner._log_worker_info(times=times, duration=duration,
concurrency=concurrency, timeout=timeout, cls=cls,
method_name=method_name, args=args)
if timeout:
timeout_queue = Queue.Queue()
collector_thr_by_timeout = threading.Thread(
target=utils.timeout_thread,
args=(timeout_queue, )
)
collector_thr_by_timeout.start()
iteration = next(iteration_gen)
start_time = time.time()
# NOTE(msimonin): keep the previous behaviour
# > when duration is 0, scenario executes exactly 1 time
current_duration = -1
while _to_be_continued(iteration, current_duration, aborted,
times=times, duration=duration):
scenario_context = runner._get_scenario_context(iteration, context)
worker_args = (
queue, cls, method_name, scenario_context, args, event_queue)
thread = threading.Thread(target=runner._worker_thread,
args=worker_args)
thread.start()
if timeout:
timeout_queue.put((thread, time.time() + timeout))
pool.append(thread)
alive_threads_in_pool += 1
while alive_threads_in_pool == concurrency:
prev_finished_threads_in_pool = finished_threads_in_pool
finished_threads_in_pool = 0
for t in pool:
if not t.is_alive():
finished_threads_in_pool += 1
alive_threads_in_pool -= finished_threads_in_pool
alive_threads_in_pool += prev_finished_threads_in_pool
if alive_threads_in_pool < concurrency:
# NOTE(boris-42): cleanup pool array. This is required because
# in other case array length will be equal to times which
# is unlimited big
while pool and not pool[0].is_alive():
pool.popleft().join()
finished_threads_in_pool -= 1
break
# we should wait to not create big noise with these checks
time.sleep(0.001)
iteration = next(iteration_gen)
current_duration = time.time() - start_time
# Wait until all threads are done
while pool:
pool.popleft().join()
if timeout:
timeout_queue.put((None, None,))
collector_thr_by_timeout.join()
@validation.configure("check_constant")
class CheckConstantValidator(validation.Validator):
"""Additional schema validation for constant runner"""
def validate(self, context, config, plugin_cls, plugin_cfg):
if plugin_cfg.get("concurrency", 1) > plugin_cfg.get("times", 1):
return self.fail(
"Parameter 'concurrency' means a number of parallel "
"executions of iterations. Parameter 'times' means total "
"number of iteration executions. It is redundant "
"(and restricted) to have number of parallel iterations "
"bigger then total number of iterations.")
@validation.add("check_constant")
@runner.configure(name="constant")
class ConstantScenarioRunner(runner.ScenarioRunner):
"""Creates constant load executing a scenario a specified number of times.
This runner will place a constant load on the cloud under test by
executing each scenario iteration without pausing between iterations
up to the number of times specified in the scenario config.
The concurrency parameter of the scenario config controls the
number of concurrent iterations which execute during a single
scenario in order to simulate the activities of multiple users
placing load on the cloud under test.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"concurrency": {
"type": "integer",
"minimum": 1,
"description": "The number of parallel iteration executions."
},
"times": {
"type": "integer",
"minimum": 1,
"description": "Total number of iteration executions."
},
"timeout": {
"type": "number",
"description": "Operation's timeout."
},
"max_cpu_count": {
"type": "integer",
"minimum": 1,
"description": "The maximum number of processes to create load"
" from."
}
},
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
This method generates a constant load on the cloud under test by
executing each scenario iteration using a pool of processes without
pausing between iterations up to the number of times specified
in the scenario config.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
timeout = self.config.get("timeout", 0) # 0 means no timeout
times = self.config.get("times", 1)
concurrency = self.config.get("concurrency", 1)
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
processes_to_start = min(max_cpu_used, times, concurrency)
concurrency_per_worker, concurrency_overhead = divmod(
concurrency, processes_to_start)
self._log_debug_info(times=times, concurrency=concurrency,
timeout=timeout, max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(concurrency_overhead):
while True:
yield (result_queue, iteration_gen, timeout,
concurrency_per_worker + (concurrency_overhead and 1),
times, None, context, cls, method_name, args,
event_queue, self.aborted)
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
@runner.configure(name="constant_for_duration")
class ConstantForDurationScenarioRunner(runner.ScenarioRunner):
"""Creates constant load executing a scenario for an interval of time.
This runner will place a constant load on the cloud under test by
executing each scenario iteration without pausing between iterations
until a specified interval of time has elapsed.
The concurrency parameter of the scenario config controls the
number of concurrent iterations which execute during a single
sceanario in order to simulate the activities of multiple users
placing load on the cloud under test.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"concurrency": {
"type": "integer",
"minimum": 1,
"description": "The number of parallel iteration executions."
},
"duration": {
"type": "number",
"minimum": 0.0,
"description": "The number of seconds during which to generate"
" a load. If the duration is 0, the scenario"
" will run once per parallel execution."
},
"timeout": {
"type": "number",
"minimum": 1,
"description": "Operation's timeout."
}
},
"required": ["duration"],
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
This method generates a constant load on the cloud under test by
executing each scenario iteration using a pool of processes without
pausing between iterations up to the number of times specified
in the scenario config.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
timeout = self.config.get("timeout", 600)
duration = self.config.get("duration", 0)
concurrency = self.config.get("concurrency", 1)
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
processes_to_start = min(max_cpu_used, concurrency)
concurrency_per_worker, concurrency_overhead = divmod(
concurrency, processes_to_start)
self._log_debug_info(duration=duration, concurrency=concurrency,
timeout=timeout, max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(concurrency_overhead):
while True:
yield (result_queue, iteration_gen, timeout,
concurrency_per_worker + (concurrency_overhead and 1),
None, duration, context, cls, method_name, args,
event_queue, self.aborted)
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)

296
rally/plugins/task/runners/rps.py Normal file
View File

@ -0,0 +1,296 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import multiprocessing
import queue as Queue
import threading
import time
from rally.common import logging
from rally.common import utils
from rally.common import validation
from rally import consts
from rally.task import runner
LOG = logging.getLogger(__name__)
def _worker_process(queue, iteration_gen, timeout, times, max_concurrent,
context, cls, method_name, args, event_queue, aborted,
runs_per_second, rps_cfg, processes_to_start, info):
"""Start scenario within threads.
Spawn N threads per second. Each thread runs the scenario once, and appends
result to queue. A maximum of max_concurrent threads will be ran
concurrently.
:param queue: queue object to append results
:param iteration_gen: next iteration number generator
:param timeout: operation's timeout
:param times: total number of scenario iterations to be run
:param max_concurrent: maximum worker concurrency
:param context: scenario context object
:param cls: scenario class
:param method_name: scenario method name
:param args: scenario args
:param aborted: multiprocessing.Event that aborts load generation if
the flag is set
:param runs_per_second: function that should return desired rps value
:param rps_cfg: rps section from task config
:param processes_to_start: int, number of started processes for scenario
execution
:param info: info about all processes count and counter of runned process
"""
pool = collections.deque()
if isinstance(rps_cfg, dict):
rps = rps_cfg["start"]
else:
rps = rps_cfg
sleep = 1.0 / rps
runner._log_worker_info(times=times, rps=rps, timeout=timeout,
cls=cls, method_name=method_name, args=args)
time.sleep(
(sleep * info["processes_counter"]) / info["processes_to_start"])
start = time.time()
timeout_queue = Queue.Queue()
if timeout:
collector_thr_by_timeout = threading.Thread(
target=utils.timeout_thread,
args=(timeout_queue, )
)
collector_thr_by_timeout.start()
i = 0
while i < times and not aborted.is_set():
scenario_context = runner._get_scenario_context(next(iteration_gen),
context)
worker_args = (
queue, cls, method_name, scenario_context, args, event_queue)
thread = threading.Thread(target=runner._worker_thread,
args=worker_args)
i += 1
thread.start()
if timeout:
timeout_queue.put((thread, time.time() + timeout))
pool.append(thread)
time_gap = time.time() - start
real_rps = i / time_gap if time_gap else "Infinity"
LOG.debug(
"Worker: %s rps: %s (requested rps: %s)" %
(i, real_rps, runs_per_second(rps_cfg, start, processes_to_start)))
# try to join latest thread(s) until it finished, or until time to
# start new thread (if we have concurrent slots available)
while i / (time.time() - start) > runs_per_second(
rps_cfg, start, processes_to_start) or (
len(pool) >= max_concurrent):
if pool:
pool[0].join(0.001)
if not pool[0].is_alive():
pool.popleft()
else:
time.sleep(0.001)
while pool:
pool.popleft().join()
if timeout:
timeout_queue.put((None, None,))
collector_thr_by_timeout.join()
@validation.configure("check_rps")
class CheckPRSValidator(validation.Validator):
"""Additional schema validation for rps runner"""
def validate(self, context, config, plugin_cls, plugin_cfg):
if isinstance(plugin_cfg["rps"], dict):
if plugin_cfg["rps"]["end"] < plugin_cfg["rps"]["start"]:
msg = "rps end value must not be less than rps start value."
return self.fail(msg)
@validation.add("check_rps")
@runner.configure(name="rps")
class RPSScenarioRunner(runner.ScenarioRunner):
"""Scenario runner that does the job with specified frequency.
Every single scenario iteration is executed with specified frequency
(runs per second) in a pool of processes. The scenario will be
launched for a fixed number of times in total (specified in the config).
An example of a rps scenario is booting 1 VM per second. This
execution type is thus very helpful in understanding the maximal load that
a certain cloud can handle.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"times": {
"type": "integer",
"minimum": 1
},
"rps": {
"anyOf": [
{
"description": "Generate constant requests per second "
"during the whole workload.",
"type": "number",
"exclusiveMinimum": 0,
"minimum": 0
},
{
"type": "object",
"description": "Increase requests per second for "
"specified value each time after a "
"certain number of seconds.",
"properties": {
"start": {
"type": "number",
"minimum": 1
},
"end": {
"type": "number",
"minimum": 1
},
"step": {
"type": "number",
"minimum": 1
},
"duration": {
"type": "number",
"minimum": 1
}
},
"additionalProperties": False,
"required": ["start", "end", "step"]
}
],
},
"timeout": {
"type": "number",
},
"max_concurrency": {
"type": "integer",
"minimum": 1
},
"max_cpu_count": {
"type": "integer",
"minimum": 1
}
},
"required": ["times", "rps"],
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
Every single scenario iteration is executed with specified
frequency (runs per second) in a pool of processes. The scenario is
launched for a fixed number of times in total (specified in the
config).
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: Context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config["times"]
timeout = self.config.get("timeout", 0) # 0 means no timeout
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
def runs_per_second(rps_cfg, start_timer, number_of_processes):
"""At the given second return desired rps."""
if not isinstance(rps_cfg, dict):
return float(rps_cfg) / number_of_processes
stage_order = (time.time() - start_timer) / rps_cfg.get(
"duration", 1) - 1
rps = (float(rps_cfg["start"] + rps_cfg["step"] * stage_order)
/ number_of_processes)
return min(rps, float(rps_cfg["end"]))
processes_to_start = min(max_cpu_used, times,
self.config.get("max_concurrency", times))
times_per_worker, times_overhead = divmod(times, processes_to_start)
# Determine concurrency per worker
concurrency_per_worker, concurrency_overhead = divmod(
self.config.get("max_concurrency", times), processes_to_start)
self._log_debug_info(times=times, timeout=timeout,
max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
times_per_worker=times_per_worker,
times_overhead=times_overhead,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(times_overhead, concurrency_overhead):
"""Generate arguments for process worker.
Remainder of threads per process division is distributed to
process workers equally - one thread per each process worker
until the remainder equals zero. The same logic is applied
to concurrency overhead.
:param times_overhead: remaining number of threads to be
distributed to workers
:param concurrency_overhead: remaining number of maximum
concurrent threads to be
distributed to workers
"""
while True:
yield (
result_queue, iteration_gen, timeout,
times_per_worker + (times_overhead and 1),
concurrency_per_worker + (concurrency_overhead and 1),
context, cls, method_name, args, event_queue,
self.aborted, runs_per_second, self.config["rps"],
processes_to_start
)
if times_overhead:
times_overhead -= 1
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(times_overhead, concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)

77
rally/plugins/task/runners/serial.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally.common import utils as rutils
from rally import consts
from rally.task import runner
@runner.configure(name="serial")
class SerialScenarioRunner(runner.ScenarioRunner):
"""Scenario runner that executes scenarios serially.
Unlike scenario runners that execute in parallel, the serial scenario
runner executes scenarios one-by-one in the same python interpreter process
as Rally. This allows you to execute scenario without introducing
any concurrent operations as well as interactively debug the scenario
from the same command that you use to start Rally.
"""
# NOTE(mmorais): additionalProperties is set True to allow switching
# between parallel and serial runners by modifying only *type* property
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"times": {
"type": "integer",
"minimum": 1
}
},
"additionalProperties": True
}
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
The scenario iterations are executed one-by-one in the same python
interpreter process as Rally. This allows you to execute
scenario without introducing any concurrent operations as well as
interactively debug the scenario from the same command that you use
to start Rally.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config.get("times", 1)
event_queue = rutils.DequeAsQueue(self.event_queue)
for i in range(times):
if self.aborted.is_set():
break
result = runner._run_scenario_once(
cls, method_name, runner._get_scenario_context(i, context),
args, event_queue)
self._send_result(result)
self._flush_results()

0
tests/unit/plugins/common/hook/triggers/__init__.py → rally/plugins/task/scenarios/__init__.py
View File

0
tests/unit/plugins/common/runners/__init__.py → rally/plugins/task/scenarios/dummy/__init__.py
View File

0
rally/plugins/common/scenarios/dummy/dummy.py → rally/plugins/task/scenarios/dummy/dummy.py
View File

0
tests/unit/plugins/common/scenarios/__init__.py → rally/plugins/task/scenarios/requests/__init__.py
View File

56
rally/plugins/task/scenarios/requests/http_requests.py Normal file
View File

@ -0,0 +1,56 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
from rally.plugins.task.scenarios.requests import utils
from rally.task import scenario
"""Scenarios for HTTP requests."""
@scenario.configure(name="HttpRequests.check_request")
class HttpRequestsCheckRequest(utils.RequestScenario):
def run(self, url, method, status_code, **kwargs):
"""Standard way for testing web services using HTTP requests.
This scenario is used to make request and check it with expected
Response.
:param url: url for the Request object
:param method: method for the Request object
:param status_code: expected response code
:param kwargs: optional additional request parameters
"""
self._check_request(url, method, status_code, **kwargs)
@scenario.configure(name="HttpRequests.check_random_request")
class HttpRequestsCheckRandomRequest(utils.RequestScenario):
def run(self, requests, status_code):
"""Executes random HTTP requests from provided list.
This scenario takes random url from list of requests, and raises
exception if the response is not the expected response.
:param requests: List of request dicts
:param status_code: Expected Response Code it will
be used only if we doesn't specified it in request proper
"""
request = random.choice(requests)
request.setdefault("status_code", status_code)
self._check_request(**request)

38
rally/plugins/task/scenarios/requests/utils.py Normal file
View File

@ -0,0 +1,38 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import requests
from rally.task import atomic
from rally.task import scenario
class RequestScenario(scenario.Scenario):
"""Base class for Request scenarios with basic atomic actions."""
@atomic.action_timer("requests.check_request")
def _check_request(self, url, method, status_code, **kwargs):
"""Compare request status code with specified code
:param status_code: Expected status code of request
:param url: Uniform resource locator
:param method: Type of request method (GET | POST ..)
:param kwargs: Optional additional request parameters
:raises ValueError: if return http status code
not equal to expected status code
"""
resp = requests.request(method, url, **kwargs)
if status_code != resp.status_code:
error_msg = "Expected HTTP request code is `%s` actual `%s`"
raise ValueError(
error_msg % (status_code, resp.status_code))

0
tests/unit/plugins/common/scenarios/dummy/__init__.py → rally/plugins/task/sla/__init__.py
View File

67
rally/plugins/task/sla/failure_rate.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally import consts
from rally.task import sla
@sla.configure(name="failure_rate")
class FailureRate(sla.SLA):
"""Failure rate minimum and maximum in percents."""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"properties": {
"min": {"type": "number", "minimum": 0.0, "maximum": 100.0},
"max": {"type": "number", "minimum": 0.0, "maximum": 100.0}
},
"minProperties": 1,
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(FailureRate, self).__init__(criterion_value)
self.min_percent = self.criterion_value.get("min", 0)
self.max_percent = self.criterion_value.get("max", 100)
self.errors = 0
self.total = 0
self.error_rate = 0.0
def add_iteration(self, iteration):
self.total += 1
if iteration["error"]:
self.errors += 1
self.error_rate = self.errors * 100.0 / self.total
self.success = self.min_percent <= self.error_rate <= self.max_percent
return self.success
def merge(self, other):
self.total += other.total
self.errors += other.errors
if self.total:
self.error_rate = self.errors * 100.0 / self.total
self.success = self.min_percent <= self.error_rate <= self.max_percent
return self.success
def details(self):
return ("Failure rate criteria %.2f%% <= %.2f%% <= %.2f%% - %s" %
(self.min_percent, self.error_rate,
self.max_percent, self.status()))

53
rally/plugins/task/sla/iteration_time.py Normal file
View File

@ -0,0 +1,53 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally import consts
from rally.task import sla
@sla.configure(name="max_seconds_per_iteration")
class IterationTime(sla.SLA):
"""Maximum time for one iteration in seconds."""
CONFIG_SCHEMA = {
"type": "number",
"$schema": consts.JSON_SCHEMA7,
"minimum": 0.0,
"exclusiveMinimum": 0.0}
def __init__(self, criterion_value):
super(IterationTime, self).__init__(criterion_value)
self.max_iteration_time = 0.0
def add_iteration(self, iteration):
if iteration["duration"] > self.max_iteration_time:
self.max_iteration_time = iteration["duration"]
self.success = self.max_iteration_time <= self.criterion_value
return self.success
def merge(self, other):
if other.max_iteration_time > self.max_iteration_time:
self.max_iteration_time = other.max_iteration_time
self.success = self.max_iteration_time <= self.criterion_value
return self.success
def details(self):
return ("Maximum seconds per iteration %.2fs <= %.2fs - %s" %
(self.max_iteration_time, self.criterion_value, self.status()))

56
rally/plugins/task/sla/max_average_duration.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
@sla.configure(name="max_avg_duration")
class MaxAverageDuration(sla.SLA):
"""Maximum average duration of one iteration in seconds."""
CONFIG_SCHEMA = {
"type": "number",
"$schema": consts.JSON_SCHEMA7,
"exclusiveMinimum": 0.0
}
def __init__(self, criterion_value):
super(MaxAverageDuration, self).__init__(criterion_value)
self.avg = 0.0
self.avg_comp = streaming_algorithms.MeanComputation()
def add_iteration(self, iteration):
if not iteration.get("error"):
self.avg_comp.add(iteration["duration"])
self.avg = self.avg_comp.result()
self.success = self.avg <= self.criterion_value
return self.success
def merge(self, other):
self.avg_comp.merge(other.avg_comp)
self.avg = self.avg_comp.result() or 0.0
self.success = self.avg <= self.criterion_value
return self.success
def details(self):
return ("Average duration of one iteration %.2fs <= %.2fs - %s" %
(self.avg, self.criterion_value, self.status()))

73
rally/plugins/task/sla/max_average_duration_per_atomic.py Normal file
View File

@ -0,0 +1,73 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
import collections
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
@sla.configure(name="max_avg_duration_per_atomic")
class MaxAverageDurationPerAtomic(sla.SLA):
"""Maximum average duration of one iterations atomic actions in seconds."""
CONFIG_SCHEMA = {"type": "object", "$schema": consts.JSON_SCHEMA,
"patternProperties": {".*": {
"type": "number",
"description": "The name of atomic action."}},
"minProperties": 1,
"additionalProperties": False}
def __init__(self, criterion_value):
super(MaxAverageDurationPerAtomic, self).__init__(criterion_value)
self.avg_by_action = collections.defaultdict(float)
self.avg_comp_by_action = collections.defaultdict(
streaming_algorithms.MeanComputation)
self.criterion_items = self.criterion_value.items()
def add_iteration(self, iteration):
if not iteration.get("error"):
for action in iteration["atomic_actions"]:
duration = action["finished_at"] - action["started_at"]
self.avg_comp_by_action[action["name"]].add(duration)
result = self.avg_comp_by_action[action["name"]].result()
self.avg_by_action[action["name"]] = result
self.success = all(self.avg_by_action[atom] <= val
for atom, val in self.criterion_items)
return self.success
def merge(self, other):
for atom, comp in self.avg_comp_by_action.items():
if atom in other.avg_comp_by_action:
comp.merge(other.avg_comp_by_action[atom])
self.avg_by_action = {a: comp.result() or 0.0
for a, comp in self.avg_comp_by_action.items()}
self.success = all(self.avg_by_action[atom] <= val
for atom, val in self.criterion_items)
return self.success
def details(self):
strs = ["Action: '%s'. %.2fs <= %.2fs" %
(atom, self.avg_by_action[atom], val)
for atom, val in self.criterion_items]
head = "Average duration of one iteration for atomic actions:"
end = "Status: %s" % self.status()
return "\n".join([head] + strs + [end])

111
rally/plugins/task/sla/outliers.py Normal file
View File

@ -0,0 +1,111 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
@sla.configure(name="outliers")
class Outliers(sla.SLA):
"""Limit the number of outliers (iterations that take too much time).
The outliers are detected automatically using the computation of the mean
and standard deviation (std) of the data.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"max": {"type": "integer", "minimum": 0},
"min_iterations": {"type": "integer", "minimum": 3},
"sigmas": {"type": "number", "minimum": 0.0,
"exclusiveMinimum": 0.0}
},
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(Outliers, self).__init__(criterion_value)
self.max_outliers = self.criterion_value.get("max", 0)
# NOTE(msdubov): Having 3 as default is reasonable (need enough data).
self.min_iterations = self.criterion_value.get("min_iterations", 3)
self.sigmas = self.criterion_value.get("sigmas", 3.0)
self.iterations = 0
self.outliers = 0
self.threshold = None
self.mean_comp = streaming_algorithms.MeanComputation()
self.std_comp = streaming_algorithms.StdDevComputation()
def add_iteration(self, iteration):
# NOTE(ikhudoshyn): This method can not be implemented properly.
# After adding a new iteration, both mean and standard deviation
# may change. Hence threshold will change as well. In this case we
# should again compare durations of all accounted iterations
# to the threshold. Unfortunately we can not do it since
# we do not store durations.
# Implementation provided here only gives rough approximation
# of outliers number.
if not iteration.get("error"):
duration = iteration["duration"]
self.iterations += 1
# NOTE(msdubov): First check if the current iteration is an outlier
if (self.iterations >= self.min_iterations
and self.threshold and duration > self.threshold):
self.outliers += 1
# NOTE(msdubov): Then update the threshold value
self.mean_comp.add(duration)
self.std_comp.add(duration)
if self.iterations >= 2:
mean = self.mean_comp.result()
std = self.std_comp.result()
self.threshold = mean + self.sigmas * std
self.success = self.outliers <= self.max_outliers
return self.success
def merge(self, other):
# NOTE(ikhudoshyn): This method can not be implemented properly.
# After merge, both mean and standard deviation may change.
# Hence threshold will change as well. In this case we
# should again compare durations of all accounted iterations
# to the threshold. Unfortunately we can not do it since
# we do not store durations.
# Implementation provided here only gives rough approximation
# of outliers number.
self.iterations += other.iterations
self.outliers += other.outliers
self.mean_comp.merge(other.mean_comp)
self.std_comp.merge(other.std_comp)
if self.iterations >= 2:
mean = self.mean_comp.result()
std = self.std_comp.result()
self.threshold = mean + self.sigmas * std
self.success = self.outliers <= self.max_outliers
return self.success
def details(self):
return ("Maximum number of outliers %i <= %i - %s" %
(self.outliers, self.max_outliers, self.status()))

72
rally/plugins/task/sla/performance_degradation.py Normal file
View File

@ -0,0 +1,72 @@
# Copyright 2016: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SLA (Service-level agreement) is set of details for determining compliance
with contracted values such as maximum error rate or minimum response time.
"""
from __future__ import division
from rally.common import streaming_algorithms
from rally import consts
from rally.task import sla
from rally.utils import strutils
@sla.configure(name="performance_degradation")
class PerformanceDegradation(sla.SLA):
"""Calculates performance degradation based on iteration time
This SLA plugin finds minimum and maximum duration of
iterations completed without errors during Rally task execution.
Assuming that minimum duration is 100%, it calculates
performance degradation against maximum duration.
"""
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA7,
"properties": {
"max_degradation": {
"type": "number",
"minimum": 0.0,
},
},
"required": [
"max_degradation",
],
"additionalProperties": False,
}
def __init__(self, criterion_value):
super(PerformanceDegradation, self).__init__(criterion_value)
self.max_degradation = self.criterion_value["max_degradation"]
self.degradation = streaming_algorithms.DegradationComputation()
def add_iteration(self, iteration):
if not iteration.get("error"):
self.degradation.add(iteration["duration"])
self.success = self.degradation.result() <= self.max_degradation
return self.success
def merge(self, other):
self.degradation.merge(other.degradation)
self.success = self.degradation.result() <= self.max_degradation
return self.success
def details(self):
res = strutils.format_float_to_str(self.degradation.result() or 0.0)
return "Current degradation: %s%% - %s" % (res, self.status())

71
rally/plugins/task/types.py Normal file
View File

@ -0,0 +1,71 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import requests
from rally.common.plugin import plugin
from rally import exceptions
from rally.task import types
@plugin.configure(name="path_or_url")
class PathOrUrl(types.ResourceType):
"""Check whether file exists or url available."""
def pre_process(self, resource_spec, config):
path = os.path.expanduser(resource_spec)
if os.path.isfile(path):
return path
try:
head = requests.head(path, verify=False, allow_redirects=True)
if head.status_code == 200:
return path
raise exceptions.InvalidScenarioArgument(
"Url %s unavailable (code %s)" % (path, head.status_code))
except Exception as ex:
raise exceptions.InvalidScenarioArgument(
"Url error %s (%s)" % (path, ex))
@plugin.configure(name="file")
class FileType(types.ResourceType):
"""Return content of the file by its path."""
def pre_process(self, resource_spec, config):
with open(os.path.expanduser(resource_spec), "r") as f:
return f.read()
@plugin.configure(name="expand_user_path")
class ExpandUserPath(types.ResourceType):
"""Expands user path."""
def pre_process(self, resource_spec, config):
return os.path.expanduser(resource_spec)
@plugin.configure(name="file_dict")
class FileTypeDict(types.ResourceType):
"""Return the dictionary of items with file path and file content."""
def pre_process(self, resource_spec, config):
file_type_dict = {}
for file_path in resource_spec:
file_path = os.path.expanduser(file_path)
with open(file_path, "r") as f:
file_type_dict[file_path] = f.read()
return file_type_dict

463
rally/plugins/verification/reporters.py Normal file
View File

@ -0,0 +1,463 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import json
import re
from rally.common.io import junit
from rally import consts
from rally.ui import utils as ui_utils
from rally.verification import reporter
SKIP_RE = re.compile(r"Skipped until Bug: ?(?P<bug_number>\d+) is resolved.")
LP_BUG_LINK = "https://launchpad.net/bugs/%s"
TIME_FORMAT = consts.TimeFormat.ISO8601
@reporter.configure("json")
class JSONReporter(reporter.VerificationReporter):
"""Generates verification report in JSON format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: json
{"verifications": {
"verification-uuid-1": {
"status": "finished",
"skipped": 1,
"started_at": "2001-01-01T00:00:00",
"finished_at": "2001-01-01T00:05:00",
"tests_duration": 5,
"run_args": {
"pattern": "set=smoke",
"xfail_list": {"some.test.TestCase.test_xfail":
"Some reason why it is expected."},
"skip_list": {"some.test.TestCase.test_skipped":
"This test was skipped intentionally"},
},
"success": 1,
"expected_failures": 1,
"tests_count": 3,
"failures": 0,
"unexpected_success": 0
},
"verification-uuid-2": {
"status": "finished",
"skipped": 1,
"started_at": "2002-01-01T00:00:00",
"finished_at": "2002-01-01T00:05:00",
"tests_duration": 5,
"run_args": {
"pattern": "set=smoke",
"xfail_list": {"some.test.TestCase.test_xfail":
"Some reason why it is expected."},
"skip_list": {"some.test.TestCase.test_skipped":
"This test was skipped intentionally"},
},
"success": 1,
"expected_failures": 1,
"tests_count": 3,
"failures": 1,
"unexpected_success": 0
}
},
"tests": {
"some.test.TestCase.test_foo[tag1,tag2]": {
"name": "some.test.TestCase.test_foo",
"tags": ["tag1","tag2"],
"by_verification": {
"verification-uuid-1": {
"status": "success",
"duration": "1.111"
},
"verification-uuid-2": {
"status": "success",
"duration": "22.222"
}
}
},
"some.test.TestCase.test_skipped[tag1]": {
"name": "some.test.TestCase.test_skipped",
"tags": ["tag1"],
"by_verification": {
"verification-uuid-1": {
"status": "skipped",
"duration": "0",
"details": "Skipped until Bug: 666 is resolved."
},
"verification-uuid-2": {
"status": "skipped",
"duration": "0",
"details": "Skipped until Bug: 666 is resolved."
}
}
},
"some.test.TestCase.test_xfail": {
"name": "some.test.TestCase.test_xfail",
"tags": [],
"by_verification": {
"verification-uuid-1": {
"status": "xfail",
"duration": "3",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
},
"verification-uuid-2": {
"status": "xfail",
"duration": "3",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
}
}
},
"some.test.TestCase.test_failed": {
"name": "some.test.TestCase.test_failed",
"tags": [],
"by_verification": {
"verification-uuid-2": {
"status": "fail",
"duration": "4",
"details": "Some reason why it is expected.\\n\\n"
"Traceback (most recent call last): \\n"
" File "fake.py", line 13, in <module>\\n"
" yyy()\\n"
" File "fake.py", line 11, in yyy\\n"
" xxx()\\n"
" File "fake.py", line 8, in xxx\\n"
" bar()\\n"
" File "fake.py", line 5, in bar\\n"
" foo()\\n"
" File "fake.py", line 2, in foo\\n"
" raise Exception()\\n"
"Exception"
}
}
}
}
}
"""
@classmethod
def validate(cls, output_destination):
"""Validate destination of report.
:param output_destination: Destination of report
"""
# nothing to check :)
pass
def _generate(self):
"""Prepare raw report."""
verifications = collections.OrderedDict()
tests = {}
for v in self.verifications:
verifications[v.uuid] = {
"started_at": v.created_at.strftime(TIME_FORMAT),
"finished_at": v.updated_at.strftime(TIME_FORMAT),
"status": v.status,
"run_args": v.run_args,
"tests_count": v.tests_count,
"tests_duration": v.tests_duration,
"skipped": v.skipped,
"success": v.success,
"expected_failures": v.expected_failures,
"unexpected_success": v.unexpected_success,
"failures": v.failures,
}
for test_id, result in v.tests.items():
if test_id not in tests:
# NOTE(ylobankov): It is more convenient to see test ID
# at the first place in the report.
tags = sorted(result.get("tags", []), reverse=True,
key=lambda tag: tag.startswith("id-"))
tests[test_id] = {"tags": tags,
"name": result["name"],
"by_verification": {}}
tests[test_id]["by_verification"][v.uuid] = {
"status": result["status"],
"duration": result["duration"]
}
reason = result.get("reason", "")
if reason:
match = SKIP_RE.match(reason)
if match:
link = LP_BUG_LINK % match.group("bug_number")
reason = re.sub(match.group("bug_number"), link,
reason)
traceback = result.get("traceback", "")
sep = "\n\n" if reason and traceback else ""
d = (reason + sep + traceback.strip()) or None
if d:
tests[test_id]["by_verification"][v.uuid]["details"] = d
return {"verifications": verifications, "tests": tests}
def generate(self):
raw_report = json.dumps(self._generate(), indent=4)
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
@reporter.configure("html")
class HTMLReporter(JSONReporter):
"""Generates verification report in HTML format."""
INCLUDE_LIBS = False
# "T" separator of ISO 8601 is not user-friendly enough.
TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
def generate(self):
report = self._generate()
uuids = report["verifications"].keys()
show_comparison_note = False
for test in report["tests"].values():
# make as much as possible processing here to reduce processing
# at JS side
test["has_details"] = False
for test_info in test["by_verification"].values():
if "details" not in test_info:
test_info["details"] = None
elif not test["has_details"]:
test["has_details"] = True
durations = []
# iter by uuids to store right order for comparison
for uuid in uuids:
if uuid in test["by_verification"]:
durations.append(test["by_verification"][uuid]["duration"])
if float(durations[-1]) < 0.001:
durations[-1] = "0"
# not to display such little duration in the report
test["by_verification"][uuid]["duration"] = ""
if len(durations) > 1 and not (
durations[0] == "0" and durations[-1] == "0"):
# compare result with result of the first verification
diff = float(durations[-1]) - float(durations[0])
result = "%s (" % durations[-1]
if diff >= 0:
result += "+"
result += "%s)" % diff
test["by_verification"][uuid]["duration"] = result
if not show_comparison_note and len(durations) > 2:
# NOTE(andreykurilin): only in case of comparison of more
# than 2 results of the same test we should display a note
# about the comparison strategy
show_comparison_note = True
template = ui_utils.get_template("verification/report.html")
context = {"uuids": list(uuids),
"verifications": report["verifications"],
"tests": report["tests"],
"show_comparison_note": show_comparison_note}
raw_report = template.render(data=json.dumps(context),
include_libs=self.INCLUDE_LIBS)
# in future we will support html_static and will need to save more
# files
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}
@reporter.configure("html-static")
class HTMLStaticReporter(HTMLReporter):
"""Generates verification report in HTML format with embedded JS/CSS."""
INCLUDE_LIBS = True
@reporter.configure("junit-xml")
class JUnitXMLReporter(reporter.VerificationReporter):
"""Generates verification report in JUnit-XML format.
An example of the report (All dates, numbers, names appearing in this
example are fictitious. Any resemblance to real things is purely
coincidental):
.. code-block:: xml
<testsuites>
<!--Report is generated by Rally 0.8.0 at 2002-01-01T00:00:00-->
<testsuite id="verification-uuid-1"
tests="9"
time="1.111"
errors="0"
failures="3"
skipped="0"
timestamp="2001-01-01T00:00:00">
<testcase classname="some.test.TestCase"
name="test_foo"
time="8"
timestamp="2001-01-01T00:01:00" />
<testcase classname="some.test.TestCase"
name="test_skipped"
time="0"
timestamp="2001-01-01T00:02:00">
<skipped>Skipped until Bug: 666 is resolved.</skipped>
</testcase>
<testcase classname="some.test.TestCase"
name="test_xfail"
time="3"
timestamp="2001-01-01T00:03:00">
<!--It is an expected failure due to: something-->
<!--Traceback:
HEEELP-->
</testcase>
<testcase classname="some.test.TestCase"
name="test_uxsuccess"
time="3"
timestamp="2001-01-01T00:04:00">
<failure>
It is an unexpected success. The test should fail due to:
It should fail, I said!
</failure>
</testcase>
</testsuite>
<testsuite id="verification-uuid-2"
tests="99"
time="22.222"
errors="0"
failures="33"
skipped="0"
timestamp="2002-01-01T00:00:00">
<testcase classname="some.test.TestCase"
name="test_foo"
time="8"
timestamp="2001-02-01T00:01:00" />
<testcase classname="some.test.TestCase"
name="test_failed"
time="8"
timestamp="2001-02-01T00:02:00">
<failure>HEEEEEEELP</failure>
</testcase>
<testcase classname="some.test.TestCase"
name="test_skipped"
time="0"
timestamp="2001-02-01T00:03:00">
<skipped>Skipped until Bug: 666 is resolved.</skipped>
</testcase>
<testcase classname="some.test.TestCase"
name="test_xfail"
time="4"
timestamp="2001-02-01T00:04:00">
<!--It is an expected failure due to: something-->
<!--Traceback:
HEEELP-->
</testcase>
</testsuite>
</testsuites>
"""
@classmethod
def validate(cls, output_destination):
pass
def generate(self):
report = junit.JUnitXML()
for v in self.verifications:
test_suite = report.add_test_suite(
id=v.uuid,
time=str(v.tests_duration),
timestamp=v.created_at.strftime(TIME_FORMAT)
)
test_suite.setup_final_stats(
tests=str(v.tests_count),
skipped=str(v.skipped),
failures=str(v.failures + v.unexpected_success)
)
tests = sorted(v.tests.values(),
key=lambda t: (t.get("timestamp", ""), t["name"]))
for result in tests:
class_name, name = result["name"].rsplit(".", 1)
test_id = [tag[3:] for tag in result.get("tags", [])
if tag.startswith("id-")]
test_case = test_suite.add_test_case(
id=(test_id[0] if test_id else None),
time=result["duration"], name=name, classname=class_name,
timestamp=result.get("timestamp"))
if result["status"] == "success":
# nothing to add
pass
elif result["status"] == "uxsuccess":
test_case.mark_as_uxsuccess(
result.get("reason"))
elif result["status"] == "fail":
test_case.mark_as_failed(
result.get("traceback", None))
elif result["status"] == "xfail":
trace = result.get("traceback", None)
test_case.mark_as_xfail(
result.get("reason", None),
f"Traceback:\n{trace}" if trace else None)
elif result["status"] == "skip":
test_case.mark_as_skipped(
result.get("reason", None))
else:
# wtf is it?! we should add validation of results...
pass
raw_report = report.to_string()
if self.output_destination:
return {"files": {self.output_destination: raw_report},
"open": self.output_destination}
else:
return {"print": raw_report}

161
rally/plugins/verification/testr.py Normal file
View File

@ -0,0 +1,161 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import shutil
import subprocess
from rally.common.io import subunit_v2
from rally.common import logging
from rally.common import utils as common_utils
from rally import exceptions
from rally.verification import context
from rally.verification import manager
from rally.verification import utils
LOG = logging.getLogger(__name__)
TEST_NAME_RE = re.compile(r"^[a-zA-Z_.0-9]+(\[[a-zA-Z-_,=0-9]*\])?$")
@context.configure("testr", order=999)
class TestrContext(context.VerifierContext):
"""Context to transform 'run_args' into CLI arguments for testr."""
def __init__(self, ctx):
super(TestrContext, self).__init__(ctx)
self._tmp_files = []
def setup(self):
super(TestrContext, self).setup()
use_testr = getattr(self.verifier.manager, "_use_testr", True)
if use_testr:
base_cmd = "testr"
else:
base_cmd = "stestr"
self.context["testr_cmd"] = [base_cmd, "run", "--subunit"]
run_args = self.verifier.manager.prepare_run_args(
self.context.get("run_args", {}))
concurrency = run_args.get("concurrency", 0)
if concurrency == 0 or concurrency > 1:
if use_testr:
self.context["testr_cmd"].append("--parallel")
if concurrency >= 1:
if concurrency == 1 and not use_testr:
self.context["testr_cmd"].append("--serial")
else:
self.context["testr_cmd"].extend(
["--concurrency", str(concurrency)])
load_list = self.context.get("load_list")
skip_list = self.context.get("skip_list")
if skip_list:
load_list = set(load_list) - set(skip_list)
if load_list:
load_list_file = common_utils.generate_random_path()
with open(load_list_file, "w") as f:
f.write("\n".join(load_list))
self._tmp_files.append(load_list_file)
self.context["testr_cmd"].extend(["--load-list", load_list_file])
if run_args.get("failed"):
self.context["testr_cmd"].append("--failing")
if run_args.get("pattern"):
self.context["testr_cmd"].append(run_args.get("pattern"))
def cleanup(self):
for f in self._tmp_files:
if os.path.exists(f):
os.remove(f)
class TestrLauncher(manager.VerifierManager):
"""Testr/sTestr wrapper."""
def __init__(self, *args, **kwargs):
super(TestrLauncher, self).__init__(*args, **kwargs)
self._use_testr = os.path.exists(os.path.join(
self.repo_dir, ".testr.conf"))
@property
def run_environ(self):
return self.environ
def _init_testr(self):
"""Initialize testr."""
test_repository_dir = os.path.join(self.base_dir, ".testrepository")
# NOTE(andreykurilin): Is there any possibility that .testrepository
# presents in clear repo?!
if not os.path.isdir(test_repository_dir):
LOG.debug("Initializing testr.")
if self._use_testr:
base_cmd = "testr"
else:
base_cmd = "stestr"
try:
utils.check_output([base_cmd, "init"], cwd=self.repo_dir,
env=self.environ)
except (subprocess.CalledProcessError, OSError):
if os.path.exists(test_repository_dir):
shutil.rmtree(test_repository_dir)
raise exceptions.RallyException("Failed to initialize testr.")
def install(self):
super(TestrLauncher, self).install()
self._init_testr()
def list_tests(self, pattern=""):
"""List all tests."""
if self._use_testr:
cmd = ["testr", "list-tests", pattern]
else:
cmd = ["stestr", "list", pattern]
output = utils.check_output(cmd,
cwd=self.repo_dir, env=self.environ,
debug_output=False)
return [t for t in output.split("\n") if TEST_NAME_RE.match(t)]
def run(self, context):
"""Run tests."""
testr_cmd = context["testr_cmd"]
LOG.debug("Test(s) started by the command: '%s'."
% " ".join(testr_cmd))
stream = subprocess.Popen(testr_cmd, env=self.run_environ,
cwd=self.repo_dir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
xfail_list = context.get("xfail_list")
skip_list = context.get("skip_list")
results = subunit_v2.parse(stream.stdout, live=True,
expected_failures=xfail_list,
skipped_tests=skip_list,
logger_name=self.verifier.name)
stream.wait()
return results
def prepare_run_args(self, run_args):
"""Prepare 'run_args' for testr context.
This method is called by TestrContext before transforming 'run_args'
into CLI arguments for testr.
"""
return run_args

2
tests/ci/cover.sh
View File

@ -15,7 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
ALLOWED_EXTRA_MISSING=4
ALLOWED_EXTRA_MISSING=400
show_diff () {
head -1 $1

0
tests/unit/plugins/common/scenarios/requests/__init__.py → tests/unit/plugins/task/__init__.py
View File

0
tests/unit/plugins/common/sla/__init__.py → tests/unit/plugins/task/contexts/__init__.py
View File

2
tests/unit/plugins/common/contexts/test_dummy.py → tests/unit/plugins/task/contexts/test_dummy.py
View File

@ -13,7 +13,7 @@
# under the License.
from rally import exceptions
from rally.plugins.common.contexts import dummy
from rally.plugins.task.contexts import dummy
from tests.unit import test

0
tests/unit/plugins/common/verification/__init__.py → tests/unit/plugins/task/exporters/__init__.py
View File

0
tests/unit/plugins/task/exporters/elastic/__init__.py Normal file
View File

4
tests/unit/plugins/common/exporters/elastic/test_client.py → tests/unit/plugins/task/exporters/elastic/test_client.py
View File

@ -16,11 +16,11 @@ import copy
from unittest import mock
from rally import exceptions
from rally.plugins.common.exporters.elastic import client
from rally.plugins.task.exporters.elastic import client
from tests.unit import test
PATH = "rally.plugins.common.exporters.elastic.client"
PATH = "rally.plugins.task.exporters.elastic.client"
class ElasticSearchClientTestCase(test.TestCase):

4
tests/unit/plugins/common/exporters/elastic/test_exporter.py → tests/unit/plugins/task/exporters/elastic/test_exporter.py
View File

@ -19,11 +19,11 @@ from unittest import mock
import ddt
from rally import exceptions
from rally.plugins.common.exporters.elastic import exporter as elastic
from rally.plugins.task.exporters.elastic import exporter as elastic
from tests.unit import test
PATH = "rally.plugins.common.exporters.elastic.exporter"
PATH = "rally.plugins.task.exporters.elastic.exporter"
class ValidatorTestCase(test.TestCase):

2
tests/unit/plugins/common/exporters/elastic/test_flatten.py → tests/unit/plugins/task/exporters/elastic/test_flatten.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.plugins.common.exporters.elastic import flatten
from rally.plugins.task.exporters.elastic import flatten
from tests.unit import test

0
tests/unit/plugins/common/exporters/junit_report.xml → tests/unit/plugins/task/exporters/junit_report.xml
View File

4
tests/unit/plugins/common/exporters/test_html.py → tests/unit/plugins/task/exporters/test_html.py
View File

@ -15,10 +15,10 @@
import os
from unittest import mock
from rally.plugins.common.exporters import html
from rally.plugins.task.exporters import html
from tests.unit import test
PATH = "rally.plugins.common.exporters.html"
PATH = "rally.plugins.task.exporters.html"
def get_tasks_results():

6
tests/unit/plugins/common/exporters/test_json_exporter.py → tests/unit/plugins/task/exporters/test_json_exporter.py
View File

@ -17,11 +17,11 @@ import datetime as dt
from unittest import mock
from rally.common import version as rally_version
from rally.plugins.common.exporters import json_exporter
from tests.unit.plugins.common.exporters import test_html
from rally.plugins.task.exporters import json_exporter
from tests.unit.plugins.task.exporters import test_html
from tests.unit import test
PATH = "rally.plugins.common.exporters.json_exporter"
PATH = "rally.plugins.task.exporters.json_exporter"
class JSONExporterTestCase(test.TestCase):

2
tests/unit/plugins/common/exporters/test_junit.py → tests/unit/plugins/task/exporters/test_junit.py
View File

@ -16,7 +16,7 @@ import datetime as dt
import os
from unittest import mock
from rally.plugins.common.exporters import junit
from rally.plugins.task.exporters import junit
from tests.unit import test

4
tests/unit/plugins/common/exporters/test_trends.py → tests/unit/plugins/task/exporters/test_trends.py
View File

@ -15,10 +15,10 @@
import os
from unittest import mock
from rally.plugins.common.exporters import trends
from rally.plugins.task.exporters import trends
from tests.unit import test
PATH = "rally.plugins.common.exporters.html"
PATH = "rally.plugins.task.exporters.html"
def get_tasks_results():

0
tests/unit/plugins/task/hook_triggers/__init__.py Normal file
View File

2
tests/unit/plugins/common/hook/triggers/test_event.py → tests/unit/plugins/task/hook_triggers/test_event.py
View File

@ -17,7 +17,7 @@ from unittest import mock
import ddt
from rally.plugins.common.hook.triggers import event
from rally.plugins.task.hook_triggers import event
from rally.task import hook
from tests.unit import test

2
tests/unit/plugins/common/hook/triggers/test_periodic.py → tests/unit/plugins/task/hook_triggers/test_periodic.py
View File

@ -17,7 +17,7 @@ from unittest import mock
import ddt
from rally.plugins.common.hook.triggers import periodic
from rally.plugins.task.hook_triggers import periodic
from rally.task import hook
from tests.unit import test

0
tests/unit/plugins/task/hooks/__init__.py Normal file
View File

6
tests/unit/plugins/common/hook/test_sys_call.py → tests/unit/plugins/task/hooks/test_sys_call.py
View File

@ -19,7 +19,7 @@ from unittest import mock
import ddt
from rally import consts
from rally.plugins.common.hook import sys_call
from rally.plugins.task.hooks import sys_call
from rally.task import hook
from tests.unit import fakes
from tests.unit import test
@ -57,7 +57,7 @@ class SysCallHookTestCase(test.TestCase):
"title": "Bar Pie"}]}})
@ddt.unpack
@mock.patch("rally.common.utils.Timer", side_effect=fakes.FakeTimer)
@mock.patch("rally.plugins.common.hook.sys_call.subprocess.Popen")
@mock.patch("rally.plugins.task.hooks.sys_call.subprocess.Popen")
def test_run(self, mock_popen, mock_timer, stdout, expected):
popen_instance = mock_popen.return_value
popen_instance.returncode = 0
@ -88,7 +88,7 @@ class SysCallHookTestCase(test.TestCase):
"expected_data_stderr": "StdErr: (empty)"})
@ddt.unpack
@mock.patch("rally.common.utils.Timer", side_effect=fakes.FakeTimer)
@mock.patch("rally.plugins.common.hook.sys_call.subprocess.Popen")
@mock.patch("rally.plugins.task.hooks.sys_call.subprocess.Popen")
def test_run_error(self, mock_popen, mock_timer, communicate_streams,
expected_error_details, expected_data_stderr):
popen_instance = mock_popen.return_value

0
tests/unit/plugins/task/runners/__init__.py Normal file
View File

4
tests/unit/plugins/common/runners/test_constant.py → tests/unit/plugins/task/runners/test_constant.py
View File

@ -17,14 +17,14 @@ from unittest import mock
import ddt
from rally.plugins.common.runners import constant
from rally.plugins.task.runners import constant
from rally.task import runner
from tests.unit import fakes
from tests.unit import test
RUNNERS_BASE = "rally.task.runner."
RUNNERS = "rally.plugins.common.runners."
RUNNERS = "rally.plugins.task.runners."
@ddt.ddt

4
tests/unit/plugins/common/runners/test_rps.py → tests/unit/plugins/task/runners/test_rps.py
View File

@ -17,14 +17,14 @@ from unittest import mock
import ddt
from rally.plugins.common.runners import rps
from rally.plugins.task.runners import rps
from rally.task import runner
from tests.unit import fakes
from tests.unit import test
RUNNERS_BASE = "rally.task.runner."
RUNNERS = "rally.plugins.common.runners."
RUNNERS = "rally.plugins.task.runners."
@ddt.ddt

2
tests/unit/plugins/common/runners/test_serial.py → tests/unit/plugins/task/runners/test_serial.py
View File

@ -15,7 +15,7 @@
from unittest import mock
from rally.plugins.common.runners import serial
from rally.plugins.task.runners import serial
from tests.unit import fakes
from tests.unit import test

0
tests/unit/plugins/task/scenarios/__init__.py Normal file
View File

0
tests/unit/plugins/task/scenarios/dummy/__init__.py Normal file
View File

4
tests/unit/plugins/common/scenarios/dummy/test_dummy.py → tests/unit/plugins/task/scenarios/dummy/test_dummy.py
View File

@ -14,11 +14,11 @@ from unittest import mock
import ddt
from rally.plugins.common.scenarios.dummy import dummy
from rally.plugins.task.scenarios.dummy import dummy
from tests.unit import test
DUMMY = "rally.plugins.common.scenarios.dummy.dummy."
DUMMY = "rally.plugins.task.scenarios.dummy.dummy."
@ddt.ddt

0
tests/unit/plugins/task/scenarios/requests/__init__.py Normal file
View File

4
tests/unit/plugins/common/scenarios/requests/test_http_requests.py → tests/unit/plugins/task/scenarios/requests/test_http_requests.py
View File

@ -12,10 +12,10 @@
from unittest import mock
from rally.plugins.common.scenarios.requests import http_requests
from rally.plugins.task.scenarios.requests import http_requests
from tests.unit import test
SCN = "rally.plugins.common.scenarios"
SCN = "rally.plugins.task.scenarios"
class RequestScenarioTestCase(test.TestCase):

2
tests/unit/plugins/common/scenarios/requests/test_utils.py → tests/unit/plugins/task/scenarios/requests/test_utils.py
View File

@ -12,7 +12,7 @@
from unittest import mock
from rally.plugins.common.scenarios.requests import utils
from rally.plugins.task.scenarios.requests import utils
from tests.unit import test

0
tests/unit/plugins/task/sla/__init__.py Normal file
View File

2
tests/unit/plugins/common/sla/test_failure_rate.py → tests/unit/plugins/task/sla/test_failure_rate.py
View File

@ -16,7 +16,7 @@
import ddt
from rally.plugins.common.sla import failure_rate
from rally.plugins.task.sla import failure_rate
from rally.task import sla
from tests.unit import test

2
tests/unit/plugins/common/sla/test_iteration_time.py → tests/unit/plugins/task/sla/test_iteration_time.py
View File

@ -16,7 +16,7 @@
import ddt
from rally.plugins.common.sla import iteration_time
from rally.plugins.task.sla import iteration_time
from rally.task import sla
from tests.unit import test

2
tests/unit/plugins/common/sla/test_max_average_duration.py → tests/unit/plugins/task/sla/test_max_average_duration.py
View File

@ -16,7 +16,7 @@
import ddt
from rally.plugins.common.sla import max_average_duration
from rally.plugins.task.sla import max_average_duration
from rally.task import sla
from tests.unit import test

2
tests/unit/plugins/common/sla/test_max_average_duration_per_atomic.py → tests/unit/plugins/task/sla/test_max_average_duration_per_atomic.py
View File

@ -15,7 +15,7 @@
import ddt
from rally.plugins.common.sla import max_average_duration_per_atomic as madpa
from rally.plugins.task.sla import max_average_duration_per_atomic as madpa
from rally.task import sla
from tests.unit import test

2
tests/unit/plugins/common/sla/test_outliers.py → tests/unit/plugins/task/sla/test_outliers.py
View File

@ -16,7 +16,7 @@
import ddt
from rally.plugins.common.sla import outliers
from rally.plugins.task.sla import outliers
from rally.task import sla
from tests.unit import test

2
tests/unit/plugins/common/sla/test_performance_degradation.py → tests/unit/plugins/task/sla/test_performance_degradation.py
View File

@ -15,7 +15,7 @@
import ddt
from rally.plugins.common.sla import performance_degradation as perfdegr
from rally.plugins.task.sla import performance_degradation as perfdegr
from rally.task import sla
from tests.unit import test

Some files were not shown because too many files have changed in this diff Show More