Extend list benchmarks for ceilometer

Existing benchmarks which process a list of resources and meters
are updated via new context and adding new configured query
parameters.

It allows to find a weak places in requesting ceilometer
data and extend count of scenarios which use a ceilometer.

Change-Id: Ifbb77915267f7e339ea7adbd9dae34ca59acb78d
This commit is contained in:
Ilya Tyaptin 2015-10-22 14:19:58 +03:00
parent 166261c4d2
commit 76b8df58d2
15 changed files with 626 additions and 46 deletions

View File

@ -438,13 +438,21 @@
CeilometerMeters.list_meters:
-
runner:
type: "constant"
times: 10
concurrency: 10
type: constant
times: 10
concurrency: 2
context:
users:
tenants: 1
users_per_tenant: 1
users:
tenants: 1
users_per_tenant: 1
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 1
samples_per_resource: 1
timestamp_interval: 1
sla:
failure_rate:
max: 0
@ -452,13 +460,21 @@
CeilometerResource.list_resources:
-
runner:
type: "constant"
times: 10
concurrency: 10
type: constant
times: 10
concurrency: 2
context:
users:
tenants: 1
users_per_tenant: 1
users:
tenants: 1
users_per_tenant: 1
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 1
samples_per_resource: 1
timestamp_interval: 1
sla:
failure_rate:
max: 0

View File

@ -24,6 +24,39 @@ class CeilometerMeters(ceiloutils.CeilometerScenario):
@validation.required_services(consts.Service.CEILOMETER)
@validation.required_openstack(users=True)
@scenario.configure()
def list_meters(self):
"""Fetch user's meters."""
self._list_meters()
def list_meters(self, metadata_query=None, limit=None):
"""Check all available queries for list resource request.
:param metadata_query: dict with metadata fields and values
:param limit: limit of meters in response
"""
self.list_matched_meters(filter_by_project_id=True)
self.list_matched_meters(filter_by_user_id=True)
self.list_matched_meters(filter_by_resource_id=True)
if metadata_query:
self.list_matched_meters(metadata_query=metadata_query)
if limit:
self.list_matched_meters(limit=limit)
@validation.required_services(consts.Service.CEILOMETER)
@validation.required_openstack(users=True)
@scenario.configure()
def list_matched_meters(self, filter_by_user_id=False,
filter_by_project_id=False,
filter_by_resource_id=False,
metadata_query=None,
limit=None):
"""Get meters that matched fields from context and args.
:param filter_by_user_id: flag for query by user_id
:param filter_by_project_id: flag for query by project_id
:param filter_by_resource_id: flag for query by resource_id
:param metadata_query: dict with metadata fields and values for query
:param limit: count of resources in response
"""
query = self._make_general_query(filter_by_project_id,
filter_by_user_id,
filter_by_resource_id,
metadata_query)
self._list_meters(query, limit)

View File

@ -25,12 +25,33 @@ class CeilometerResource(ceiloutils.CeilometerScenario):
@validation.required_services(consts.Service.CEILOMETER)
@validation.required_openstack(users=True)
@scenario.configure()
def list_resources(self):
"""Fetch all resources.
def list_resources(self, metadata_query=None,
start_time=None,
end_time=None,
limit=None):
"""Check all available queries for list resource request.
This scenario fetches list of all resources using GET /v2/resources.
:param metadata_query: dict with metadata fields and values for query
:param start_time: lower bound of resource timestamp in isoformat
:param end_time: upper bound of resource timestamp in isoformat
:param limit: count of resources in response
"""
self._list_resources()
self.list_matched_resources(filter_by_project_id=True)
self.list_matched_resources(filter_by_user_id=True)
self.list_matched_resources(filter_by_resource_id=True)
if metadata_query:
self.list_matched_resources(metadata_query=metadata_query)
if start_time:
self.list_matched_resources(start_time=start_time)
if end_time:
self.list_matched_resources(end_time=end_time)
if start_time and end_time:
self.list_matched_resources(start_time=start_time,
end_time=end_time)
if limit:
self.list_matched_resources(limit=limit)
@validation.required_services(consts.Service.CEILOMETER)
@validation.required_openstack(users=True)
@ -48,3 +69,32 @@ class CeilometerResource(ceiloutils.CeilometerScenario):
raise exceptions.NotFoundException(message=msg)
for res_id in resources:
self._get_resource(res_id)
@validation.required_services(consts.Service.CEILOMETER)
@validation.required_openstack(users=True)
@scenario.configure()
def list_matched_resources(self, filter_by_user_id=False,
filter_by_project_id=False,
filter_by_resource_id=False,
metadata_query=None,
start_time=None,
end_time=None,
limit=None):
"""Get resources that matched fields from context and args.
:param filter_by_user_id: flag for query by user_id
:param filter_by_project_id: flag for query by project_id
:param filter_by_resource_id: flag for query by resource_id
:param metadata_query: dict with metadata fields and values for query
:param start_time: lower bound of resource timestamp in isoformat
:param end_time: upper bound of resource timestamp in isoformat
:param limit: count of resources in response
"""
query = self._make_general_query(filter_by_project_id,
filter_by_user_id,
filter_by_resource_id,
metadata_query)
query += self._make_timestamp_query(start_time, end_time)
self._list_resources(query, limit)

View File

@ -11,10 +11,12 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import six
from rally import exceptions
from rally.plugins.openstack import scenario
from rally.task import atomic
from rally.task import utils as bench_utils
@ -77,6 +79,85 @@ class CeilometerScenario(scenario.OpenStackScenario):
return samples
def _make_query_item(self, field, op="eq", value=None):
"""Create a SimpleQuery item for requests.
:param field: filtered field
:param op: operator for filtering
:param value: matched value
:returns dict with field, op and value keys for query
"""
return {"field": field, "op": op, "value": value}
def _make_general_query(self, filter_by_project_id=None,
filter_by_user_id=None,
filter_by_resource_id=None,
metadata_query=None):
"""Create a SimpleQuery for the list benchmarks.
:param filter_by_project_id: add a project id to query
:param filter_by_user_id: add a user id to query
:param filter_by_resource_id: add a resource id to query
:param metadata_query: metadata dict that will add to query
:returns SimpleQuery with specified items
"""
query = []
metadata_query = metadata_query or {}
if filter_by_user_id:
user_id = self.context["user"]["id"]
query.append(self._make_query_item("user_id", "eq", user_id))
if filter_by_project_id or filter_by_resource_id:
project_id = self.context["tenant"]["id"]
if filter_by_project_id:
query.append(self._make_query_item("project_id", "eq",
project_id))
if filter_by_resource_id:
resource_id = self.context["tenant"]["resources"][0]
query.append(self._make_query_item("resource_id", "eq",
resource_id))
for key, value in metadata_query.items():
query.append(self._make_query_item("metadata.%s" % key,
value=value))
return query
def _make_timestamp_query(self, start_time=None, end_time=None):
"""Create ceilometer query for timestamp range.
:param start_time: start datetime in isoformat
:param end_time: end datetime in isoformat
:returns query with timestamp range
"""
query = []
if end_time and start_time and end_time < start_time:
msg = "End time should be great or equal than start time"
raise exceptions.InvalidArgumentsException(msg)
if start_time:
query.append(self._make_query_item("timestamp", ">=", start_time))
if end_time:
query.append(self._make_query_item("timestamp", "<=", end_time))
return query
def _make_profiler_key(self, method, query=None, limit=None):
"""Create key for profiling method with query.
:param method: Original profiler tag for method
:param query: ceilometer query which fields will be added to key
:param limit: if it exists `limit` will be added to key
:returns profiler key that includes method and queried fields
"""
query = query or []
limit_line = limit and "limit" or ""
fields_line = "&".join("%s" % a["field"] for a in query)
key_identifiers = "&".join(x for x in (limit_line, fields_line) if x)
key = ":".join(x for x in (method, key_identifiers) if x)
return key
def _get_alarm_dict(self, **kwargs):
"""Prepare and return an alarm dict for creating an alarm.
@ -222,19 +303,6 @@ class CeilometerScenario(scenario.OpenStackScenario):
return self.admin_clients("ceilometer").trait_descriptions.list(
event_type)
@atomic.action_timer("ceilometer.list_meters")
def _list_meters(self):
"""Get list of user's meters."""
return self.clients("ceilometer").meters.list()
@atomic.action_timer("ceilometer.list_resources")
def _list_resources(self):
"""List all resources.
:returns: list of all resources
"""
return self.clients("ceilometer").resources.list()
@atomic.action_timer("ceilometer.list_samples")
def _list_samples(self):
"""List all Samples.
@ -342,3 +410,31 @@ class CeilometerScenario(scenario.OpenStackScenario):
"""
return self.clients("ceilometer").query_samples.query(
filter, orderby, limit)
def _list_resources(self, query=None, limit=None):
"""List all resources.
:param query: query list for Ceilometer api
:param limit: count of returned resources
:returns: list of all resources
"""
key = self._make_profiler_key("ceilometer.list_resources", query,
limit)
with atomic.ActionTimer(self, key):
return self.clients("ceilometer").resources.list(q=query,
limit=limit)
def _list_meters(self, query=None, limit=None):
"""Get list of user's meters.
:param query: query list for Ceilometer api
:param limit: count of returned meters
:returns: list of all meters
"""
key = self._make_profiler_key("ceilometer.list_meters", query,
limit)
with atomic.ActionTimer(self, key):
return self.clients("ceilometer").meters.list(q=query,
limit=limit)

View File

@ -0,0 +1,38 @@
{
"CeilometerMeters.list_meters": [
{
"runner": {
"type": "constant",
"times": 10,
"concurrency": 1
},
"context": {
"users": {
"tenants": 2,
"users_per_tenant": 2
},
"ceilometer": {
"counter_name": "benchmark_meter",
"counter_type": "gauge",
"counter_unit": "%",
"counter_volume": 100,
"resources_per_tenant": 100,
"samples_per_resource": 100,
"timestamp_interval": 10,
"metadata_list": [
{"status": "active", "name": "rally benchmark on",
"deleted": "false"},
{"status": "terminated", "name": "rally benchmark off",
"deleted": "true"}
]
}
},
"args": {
"limit": 50,
"metadata_query": {"status": "terminated"}
}
}
]
}

View File

@ -0,0 +1,32 @@
---
CeilometerMeters.list_meters:
-
runner:
type: constant
times: 10
concurrency: 1
context:
users:
tenants: 2
users_per_tenant: 2
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 100
samples_per_resource: 100
timestamp_interval: 10
metadata_list:
-
status: "active"
name: "rally benchmark on"
deleted: "false"
-
status: "terminated"
name: "rally benchmark off"
deleted: "true"
args:
limit: 50
metadata_query:
status: "terminated"

View File

@ -0,0 +1,37 @@
{
"CeilometerResource.list_resources": [
{
"runner": {
"type": "constant",
"times": 10,
"concurrency": 1
},
"context": {
"users": {
"tenants": 2,
"users_per_tenant": 2
},
"ceilometer": {
"counter_name": "benchmark_meter",
"counter_type": "gauge",
"counter_unit": "%",
"counter_volume": 100,
"resources_per_tenant": 100,
"samples_per_resource": 100,
"timestamp_interval": 10,
"metadata_list": [
{"status": "active", "name": "rally benchmark on",
"deleted": "false"},
{"status": "terminated", "name": "rally benchmark off",
"deleted": "true"}
]
}
},
"args": {
"limit":50,
"metadata_query": {"status": "terminated"}
}
}
]
}

View File

@ -0,0 +1,32 @@
---
CeilometerResource.list_resources:
-
runner:
type: "constant"
times: 10
concurrency: 1
context:
users:
tenants: 2
users_per_tenant: 2
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 100
samples_per_resource: 100
timestamp_interval: 10
metadata_list:
-
status: "active"
name: "rally benchmark on"
deleted: "false"
-
status: "terminated"
name: "rally benchmark off"
deleted: "true"
args:
limit: 50
metadata_query:
status: "terminated"

View File

@ -1,5 +1,5 @@
{
"CeilometerMeters.list_meters": [
"CeilometerMeters.list_matched_meters": [
{
"runner": {
"type": "constant",
@ -10,7 +10,29 @@
"users": {
"tenants": 2,
"users_per_tenant": 2
},
"ceilometer": {
"counter_name": "benchmark_meter",
"counter_type": "gauge",
"counter_unit": "%",
"counter_volume": 100,
"resources_per_tenant": 100,
"samples_per_resource": 100,
"timestamp_interval": 10,
"metadata_list": [
{"status": "active", "name": "rally benchmark on",
"deleted": "false"},
{"status": "terminated", "name": "rally benchmark off",
"deleted": "true"}
]
}
},
"args": {
"filter_by_user_id": true,
"filter_by_project_id": true,
"filter_by_resource_id": true,
"limit": 50,
"metadata_query": {"status": "terminated"}
}
}
]

View File

@ -1,11 +1,35 @@
---
CeilometerMeters.list_meters:
CeilometerMeters.list_matched_meters:
-
runner:
type: "constant"
type: constant
times: 10
concurrency: 1
context:
users:
tenants: 2
users_per_tenant: 2
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 100
samples_per_resource: 100
timestamp_interval: 10
metadata_list:
-
status: "active"
name: "rally benchmark on"
deleted: "false"
-
status: "terminated"
name: "rally benchmark off"
deleted: "true"
args:
limit: 50
filter_by_user_id: true
filter_by_project_id: true
filter_by_resource_id: true
metadata_query:
status: "terminated"

View File

@ -1,5 +1,5 @@
{
"CeilometerResource.list_resources": [
"CeilometerResource.list_matched_resources": [
{
"runner": {
"type": "constant",
@ -10,7 +10,28 @@
"users": {
"tenants": 2,
"users_per_tenant": 2
},
"ceilometer": {
"counter_name": "benchmark_meter",
"counter_type": "gauge",
"counter_unit": "%",
"counter_volume": 100,
"resources_per_tenant": 100,
"samples_per_resource": 100,
"timestamp_interval": 10,
"metadata_list": [
{"status": "active", "name": "rally benchmark on",
"deleted": "false"},
{"status": "terminated", "name": "rally benchmark off",
"deleted": "true"}
]
}
},
"args": {
"limit":50,
"metadata_query": {"status": "terminated"},
"filter_by_user_id": true,
"filter_by_project_id": true
}
}
]

View File

@ -1,6 +1,6 @@
---
CeilometerResource.list_resources:
-
CeilometerResource.list_matched_resources:
-
runner:
type: "constant"
times: 10
@ -9,4 +9,26 @@
users:
tenants: 2
users_per_tenant: 2
ceilometer:
counter_name: "benchmark_meter"
counter_type: "gauge"
counter_unit: "%"
counter_volume: 100
resources_per_tenant: 100
samples_per_resource: 100
timestamp_interval: 10
metadata_list:
-
status: "active"
name: "rally benchmark on"
deleted: "false"
-
status: "terminated"
name: "rally benchmark off"
deleted: "true"
args:
limit: 50
filter_by_user_id: true
filter_by_project_id: true
metadata_query:
status: "terminated"

View File

@ -19,8 +19,47 @@ from tests.unit import test
class CeilometerMetersTestCase(test.ScenarioTestCase):
def test_list_meters(self):
def test_all_meter_list_queries(self):
scenario = meters.CeilometerMeters(self.context)
scenario.list_matched_meters = mock.MagicMock()
metadata_query = {"a": "test"}
limit = 100
scenario.list_meters(metadata_query, limit)
scenario.list_matched_meters.assert_any_call(limit=100)
scenario.list_matched_meters.assert_any_call(
metadata_query=metadata_query)
scenario.list_matched_meters.assert_any_call(filter_by_user_id=True)
scenario.list_matched_meters.assert_any_call(filter_by_project_id=True)
scenario.list_matched_meters.assert_any_call(
filter_by_resource_id=True)
def test_meter_list_queries_without_limit_and_metadata(self):
scenario = meters.CeilometerMeters(self.context)
scenario.list_matched_meters = mock.MagicMock()
scenario.list_meters()
expected_call_args_list = [
mock.call(filter_by_project_id=True),
mock.call(filter_by_user_id=True),
mock.call(filter_by_resource_id=True)
]
self.assertSequenceEqual(expected_call_args_list,
scenario.list_matched_meters.call_args_list)
def test_list_matched_meters(self):
scenario = meters.CeilometerMeters(self.context)
scenario._list_meters = mock.MagicMock()
scenario.list_meters()
scenario._list_meters.assert_called_once_with()
context = {"user": {"tenant_id": "fake", "id": "fake_id"},
"tenant": {"id": "fake_id",
"resources": ["fake_resource"]}}
scenario.context = context
metadata_query = {"a": "test"}
limit = 100
scenario.list_matched_meters(True, True, True, metadata_query, limit)
scenario._list_meters.assert_called_once_with(
[{"field": "user_id", "value": "fake_id", "op": "eq"},
{"field": "project_id", "value": "fake_id", "op": "eq"},
{"field": "resource_id", "value": "fake_resource", "op": "eq"},
{"field": "metadata.a", "value": "test", "op": "eq"}],
100)

View File

@ -20,11 +20,54 @@ from tests.unit import test
class CeilometerResourcesTestCase(test.ScenarioTestCase):
def test_list_resources(self):
def test_all_resource_list_queries(self):
scenario = resources.CeilometerResource(self.context)
scenario.list_matched_resources = mock.MagicMock()
metadata_query = {"a": "test"}
start_time = "fake start time"
end_time = "fake end time"
limit = 100
scenario.list_resources(metadata_query, start_time,
end_time, limit)
scenario.list_matched_resources.assert_any_call(limit=100)
scenario.list_matched_resources.assert_any_call(start_time=start_time,
end_time=end_time)
scenario.list_matched_resources.assert_any_call(end_time=end_time)
scenario.list_matched_resources.assert_any_call(start_time=start_time)
scenario.list_matched_resources.assert_any_call(
metadata_query=metadata_query)
scenario.list_matched_resources.assert_any_call(filter_by_user_id=True)
scenario.list_matched_resources.assert_any_call(
filter_by_project_id=True)
scenario.list_matched_resources.assert_any_call(
filter_by_resource_id=True)
def test_list_matched_resources(self):
scenario = resources.CeilometerResource(self.context)
scenario._list_resources = mock.MagicMock()
scenario.list_resources()
scenario._list_resources.assert_called_once_with()
context = {"user": {"tenant_id": "fake", "id": "fake_id"},
"tenant": {"id": "fake_id",
"resources": ["fake_resource"]}}
scenario.context = context
metadata_query = {"a": "test"}
start_time = "2015-09-09T00:00:00"
end_time = "2015-09-10T00:00:00"
limit = 100
scenario.list_matched_resources(True, True, True, metadata_query,
start_time, end_time, limit)
scenario._list_resources.assert_called_once_with(
[{"field": "user_id", "value": "fake_id", "op": "eq"},
{"field": "project_id", "value": "fake_id", "op": "eq"},
{"field": "resource_id", "value": "fake_resource", "op": "eq"},
{"field": "metadata.a", "value": "test", "op": "eq"},
{"field": "timestamp", "value": "2015-09-09T00:00:00",
"op": ">="},
{"field": "timestamp", "value": "2015-09-10T00:00:00",
"op": "<="}
],
100)
def test_get_tenant_resources(self):
scenario = resources.CeilometerResource(self.context)
@ -37,6 +80,19 @@ class CeilometerResourcesTestCase(test.ScenarioTestCase):
for resource_id in resource_list:
scenario._get_resource.assert_any_call(resource_id)
def test_resource_list_queries_without_limit_and_metadata(self):
scenario = resources.CeilometerResource(self.context)
scenario.list_matched_resources = mock.MagicMock()
scenario.list_resources()
expected_call_args_list = [
mock.call(filter_by_project_id=True),
mock.call(filter_by_user_id=True),
mock.call(filter_by_resource_id=True)
]
self.assertSequenceEqual(
expected_call_args_list,
scenario.list_matched_resources.call_args_list)
def test_get_tenant_resources_with_exception(self):
scenario = resources.CeilometerResource(self.context)
resource_list = []

View File

@ -18,6 +18,7 @@ import datetime
from dateutil import parser
import mock
from rally import exceptions
from rally.plugins.openstack.scenarios.ceilometer import utils
from tests.unit import test
@ -46,6 +47,29 @@ class CeilometerScenarioTestCase(test.ScenarioTestCase):
parser.parse(result[1]["timestamp"])).seconds
self.assertEqual(60, samples_int)
def test__make_timestamp_query(self):
start_time = "2015-09-09T00:00:00"
end_time = "2015-09-10T00:00:00"
expected_start = [
{"field": "timestamp", "value": "2015-09-09T00:00:00",
"op": ">="}]
expected_end = [
{"field": "timestamp", "value": "2015-09-10T00:00:00",
"op": "<="}
]
actual = self.scenario._make_timestamp_query(start_time, end_time)
self.assertEqual(expected_start + expected_end, actual)
self.assertRaises(exceptions.InvalidArgumentsException,
self.scenario._make_timestamp_query,
end_time, start_time)
self.assertEqual(
expected_start,
self.scenario._make_timestamp_query(start_time=start_time))
self.assertEqual(
expected_end,
self.scenario._make_timestamp_query(end_time=end_time))
def test__list_alarms_by_id(self):
self.assertEqual(self.clients("ceilometer").alarms.get.return_value,
self.scenario._list_alarms("alarm-id"))
@ -180,7 +204,8 @@ class CeilometerScenarioTestCase(test.ScenarioTestCase):
def test__list_meters(self):
self.assertEqual(self.scenario._list_meters(),
self.clients("ceilometer").meters.list.return_value)
self.clients("ceilometer").meters.list.assert_called_once_with()
self.clients("ceilometer").meters.list.assert_called_once_with(
q=None, limit=None)
self._test_atomic_action_timer(self.scenario.atomic_actions(),
"ceilometer.list_meters")
@ -188,7 +213,8 @@ class CeilometerScenarioTestCase(test.ScenarioTestCase):
self.assertEqual(
self.scenario._list_resources(),
self.clients("ceilometer").resources.list.return_value)
self.clients("ceilometer").resources.list.assert_called_once_with()
self.clients("ceilometer").resources.list.assert_called_once_with(
q=None, limit=None)
self._test_atomic_action_timer(self.scenario.atomic_actions(),
"ceilometer.list_resources")
@ -292,3 +318,39 @@ class CeilometerScenarioTestCase(test.ScenarioTestCase):
resource_id="test-resource-id")
self._test_atomic_action_timer(self.scenario.atomic_actions(),
"ceilometer.create_sample")
def test__make_general_query(self):
self.scenario.context = {
"user": {"tenant_id": "fake", "id": "fake_id"},
"tenant": {"id": "fake_id", "resources": ["fake_resource"]}}
metadata = {"fake_field": "boo"}
expected = [
{"field": "user_id", "value": "fake_id", "op": "eq"},
{"field": "project_id", "value": "fake_id", "op": "eq"},
{"field": "resource_id", "value": "fake_resource", "op": "eq"},
{"field": "metadata.fake_field", "value": "boo", "op": "eq"},
]
actual = self.scenario._make_general_query(True, True, True, metadata)
self.assertEqual(expected, actual)
def test__make_query_item(self):
expected = {"field": "foo", "op": "eq", "value": "bar"}
self.assertEqual(expected,
self.scenario._make_query_item("foo", value="bar"))
def test__make_profiler_key(self):
query = [
{"field": "test_field1", "op": "eq", "value": "bar"},
{"field": "test_field2", "op": "==", "value": None}
]
limit = 100
method = "fake_method"
actual = self.scenario._make_profiler_key(method, query, limit)
self.assertEqual("fake_method:limit&test_field1&test_field2", actual)
actual = self.scenario._make_profiler_key(method, query, None)
self.assertEqual("fake_method:test_field1&test_field2", actual)
self.assertEqual(method,
self.scenario._make_profiler_key(method, None, None))