openstack
/
rally-openstack
Code Issues Proposed changes

Add Grafana plugin for monitoring testing 

Add testing monitoring system with push acceptor Pushgateway and metric
analyzer Grafana in conjunction with nova instance. This test case is to
check how nova instance could communicate with a monitoring system, not
only for testing monitoring availability. Suchwise Openstack platform
using is justified.

However, if the issue is to check availability of monitoring system,
push_metric_locally scenario allows to push random metric locally to
Pushgateway and just check it in Grafana.

Change-Id: I00e85189b5f54c3e6fd18fa52afe7db0eda88fed
This commit is contained in:
Peter Razumovsky 2018-04-18 13:34:50 +04:00 committed by Peter Razumovsky
parent 2a8249999f
commit 48f895b645
8 changed files with 407 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
rally_openstack/scenarios/grafana/__init__.py Normal file
View File

155
rally_openstack/scenarios/grafana/metrics.py Normal file
View File

@ -0,0 +1,155 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally.common import cfg
from rally.common import logging
from rally.task import types
from rally.task import utils
from rally.task import validation
from rally_openstack import consts
from rally_openstack import scenario
from rally_openstack.services.grafana import grafana as grafana_service
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
"""Scenarios for Pushgateway and Grafana metrics."""
@types.convert(image={"type": "glance_image"},
flavor={"type": "nova_flavor"})
@validation.add("required_services", services=[consts.Service.NOVA])
@validation.add("required_platform", platform="openstack", admin=True)
@scenario.configure(context={"cleanup@openstack": ["nova"]},
name="GrafanaMetrics.push_metric_from_instance",
platform="openstack")
class PushMetricsInstance(scenario.OpenStackScenario):
"""Test monitoring system by pushing metric from nova server and check it.
Scenario tests monitoring system, which uses Pushgateway as metric exporter
and Grafana as metrics monitoring.
The goal of the test is to check that monitoring system works correctly
with nova instance. Test case is the following: we deploy some env with
nodes on Openstack nova instances, add metric exporter (using Pushgateway
in this test) inside nodes (i.e. nova instances) for some interested
metrics (e.g. CPU, memory etc.). We want to check that metrics successfully
sends to metrics storage (e.g. Prometheus) by requesting Grafana. Create
nova instance, add Pushgateway push random metric to userdata and after
instance would be available, check Grafana datasource that pushed metric in
data.
"""
def _metric_from_instance(self, seed, image, flavor, monitor_vip,
pushgateway_port, job_name):
push_cmd = (
"echo %(seed)s 12345 | curl --data-binary "
"@- http://%(monitor_vip)s:%(pgtw_port)s/metrics/job"
"/%(job_name)s" % {"seed": seed,
"monitor_vip": monitor_vip,
"pgtw_port": pushgateway_port,
"job_name": job_name})
userdata = ("#!/bin/bash\n%s" % push_cmd)
server = self.clients("nova").servers.create(seed,
image, flavor,
userdata=userdata)
LOG.info("Server %s create started" % seed)
self.sleep_between(CONF.openstack.nova_server_boot_prepoll_delay)
utils.wait_for_status(
server,
ready_statuses=["ACTIVE"],
update_resource=utils.get_from_manager(),
timeout=CONF.openstack.nova_server_boot_timeout,
check_interval=CONF.openstack.nova_server_boot_poll_interval
)
LOG.info("Server %s with pushing metric script (metric exporter) is "
"active" % seed)
def run(self, image, flavor, monitor_vip, pushgateway_port,
grafana, datasource_id, job_name, sleep_time=5,
retries_total=30):
"""Create nova instance with pushing metric script as userdata.
Push metric to metrics storage using Pushgateway and check it in
Grafana.
:param image: image for server with userdata script
:param flavor: flavor for server with userdata script
:param monitor_vip: monitoring system IP to push metric
:param pushgateway_port: Pushgateway port to use for pushing metric
:param grafana: Grafana dict with creds and port to use for checking
metric. Format: {user: admin, password: pass, port: 9902}
:param datasource_id: metrics storage datasource ID in Grafana
:param job_name: job name to push metric in it
:param sleep_time: sleep time between checking metrics in seconds
:param retries_total: total number of retries to check metric in
Grafana
"""
seed = self.generate_random_name()
grafana_svc = grafana_service.GrafanaService(
dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port,
grafana=grafana, datasource_id=datasource_id,
job_name=job_name),
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions())
self._metric_from_instance(seed, image, flavor, monitor_vip,
pushgateway_port, job_name)
checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip,
grafana=grafana,
datasource_id=datasource_id,
sleep_time=sleep_time,
retries_total=retries_total)
self.assertTrue(checked)
@scenario.configure(name="GrafanaMetrics.push_metric_locally")
class PushMetricLocal(scenario.OpenStackScenario):
"""Test monitoring system availability with local pushing random metric."""
def run(self, monitor_vip, pushgateway_port, grafana, datasource_id,
job_name, sleep_time=5, retries_total=30):
"""Push random metric to Pushgateway locally and check it in Grafana.
:param monitor_vip: monitoring system IP to push metric
:param pushgateway_port: Pushgateway port to use for pushing metric
:param grafana: Grafana dict with creds and port to use for checking
metric. Format: {user: admin, password: pass, port: 9902}
:param datasource_id: metrics storage datasource ID in Grafana
:param job_name: job name to push metric in it
:param sleep_time: sleep time between checking metrics in seconds
:param retries_total: total number of retries to check metric in
Grafana
"""
seed = self.generate_random_name()
grafana_svc = grafana_service.GrafanaService(
dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port,
grafana=grafana, datasource_id=datasource_id,
job_name=job_name),
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions())
pushed = grafana_svc.push_metric(seed, monitor_vip=monitor_vip,
pushgateway_port=pushgateway_port,
job_name=job_name)
self.assertTrue(pushed)
checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip,
grafana=grafana,
datasource_id=datasource_id,
sleep_time=sleep_time,
retries_total=retries_total)
self.assertTrue(checked)

0
rally_openstack/services/grafana/__init__.py Normal file
View File

91
rally_openstack/services/grafana/grafana.py Normal file
View File

@ -0,0 +1,91 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import requests
from rally.common import logging
from rally.common import utils as commonutils
from rally.task import atomic
from rally.task import service
LOG = logging.getLogger(__name__)
class GrafanaService(service.Service):
def __init__(self, spec, name_generator=None, atomic_inst=None):
"""Initialization of Grafana service.
:param spec: param contains monitoring system info: IPs, ports, creds
"""
super(GrafanaService, self).__init__(None,
name_generator=name_generator,
atomic_inst=atomic_inst)
self._spec = spec
@atomic.action_timer("grafana.check_metric")
def check_metric(self, seed, sleep_time, retries_total):
"""Check metric with seed name in Grafana datasource.
:param seed: random metric name
:param sleep_time: sleep time between checking metrics in seconds
:param retries_total: total number of retries to check metric in
Grafana
:return: True if metric in Grafana datasource and False otherwise
"""
check_url = ("http://%(vip)s:%(port)s/api/datasources/proxy/:"
"%(datasource)s/api/v1/query?query=%(seed)s" % {
"vip": self._spec["monitor_vip"],
"port": self._spec["grafana"]["port"],
"datasource": self._spec["datasource_id"],
"seed": seed
})
i = 0
LOG.info("Check metric %s in Grafana" % seed)
while i < retries_total:
LOG.debug("Attempt number %s" % (i + 1))
resp = requests.get(check_url,
auth=(self._spec["grafana"]["user"],
self._spec["grafana"]["password"]))
result = resp.json()
LOG.debug("Grafana response code: %s" % resp.status_code)
if len(result["data"]["result"]) < 1 and i + 1 >= retries_total:
LOG.debug("No instance metrics found in Grafana")
return False
elif len(result["data"]["result"]) < 1:
i += 1
commonutils.interruptable_sleep(sleep_time)
else:
LOG.debug("Metric instance found in Grafana")
return True
@atomic.action_timer("grafana.push_metric")
def push_metric(self, seed):
"""Push metric by GET request using pushgateway.
:param seed: random name for metric to push
"""
push_url = "http://%(ip)s:%(port)s/metrics/job/%(job)s" % {
"ip": self._spec["monitor_vip"],
"port": self._spec["pushgateway_port"],
"job": self._spec["job_name"]
}
resp = requests.post(push_url,
headers={"Content-type": "text/xml"},
data="%s 12345\n" % seed)
if resp.ok:
LOG.info("Metric %s pushed" % seed)
else:
LOG.error("Error during push metric %s" % seed)
return resp.ok

58
samples/tasks/scenarios/grafana/push-metric-from-instance.json Normal file
View File

@ -0,0 +1,58 @@
{% set flavor_name = flavor_name or "grafana_test.small" %}
{% set image_name = image_name or "testVM" %}
{
"GrafanaMetrics.push_metric_from_instance": [
{
"args": {
"flavor": {
"name": "{{ flavor_name }}"
},
"image": {
"name": "{{ image_name }}"
},
"monitor_vip": "10.0.0.5",
"pushgateway_port": 9091,
"grafana": {
"user": "admin",
"password": "password",
"port": 3000
},
"datasource_id": 1,
"job_name": "rally_test",
"sleep_time": 5,
"retries_total": 30
},
"runner": {
"type": "constant",
"times": 10,
"concurrency": 1
},
"context": {
"users": {
"tenants": 1,
"users_per_tenant": 1
},
"flavors": [
{
"name": "{{ flavor_name }}",
"ram": 512,
"disk": 1,
"vcpus": 1
}
],
"images": {
"image_name": "{{ image_name }}",
"image_url": "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img",
"disk_format": "qcow2",
"container_format": "bare",
"visibility": "public"
}
},
"sla": {
"failure_rate": {
"max": 0
}
}
}
]
}

43
samples/tasks/scenarios/grafana/push-metric-from-instance.yaml Normal file
View File

@ -0,0 +1,43 @@
{% set flavor_name = flavor_name or "grafana_test.small" %}
{% set image_name = image_name or "testVM" %}
---
GrafanaMetrics.push_metric_from_instance:
-
args:
flavor:
name: {{ flavor_name }}
image:
name: {{ image_name }}
monitor_vip: 10.0.0.5
pushgateway_port: 9091
grafana:
user: admin
password: password
port: 3000
datasource_id: 1
job_name: rally_test
sleep_time: 5
retries_total: 30
runner:
type: "constant"
times: 10
concurrency: 1
context:
users:
tenants: 1
users_per_tenant: 1
flavors:
-
name: {{ flavor_name }}
ram: 512
disk: 1
vcpus: 1
images:
image_name: {{ image_name }}
image_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
disk_format: qcow2
container_format: bare
visibility: public
sla:
failure_rate:
max: 0

35
samples/tasks/scenarios/grafana/push-metric-locally.json Normal file
View File

@ -0,0 +1,35 @@
{
"GrafanaMetrics.push_metric_locally": [
{
"args": {
"monitor_vip": "10.0.0.5",
"pushgateway_port": 9091,
"grafana": {
"user": "admin",
"password": "password",
"port": 3000
},
"datasource_id": 1,
"job_name": "rally_test",
"sleep_time": 5,
"retries_total": 30
},
"runner": {
"type": "constant",
"times": 10,
"concurrency": 1
},
"context": {
"users": {
"tenants": 1,
"users_per_tenant": 1
}
},
"sla": {
"failure_rate": {
"max": 0
}
}
}
]
}

25
samples/tasks/scenarios/grafana/push-metric-locally.yaml Normal file
View File

@ -0,0 +1,25 @@
---
GrafanaMetrics.push_metric_locally:
-
args:
monitor_vip: 10.0.0.5
pushgateway_port: 9091
grafana:
user: admin
password: password
port: 3000
datasource_id: 1
job_name: rally_test
sleep_time: 5
retries_total: 30
runner:
type: "constant"
times: 10
concurrency: 1
context:
users:
tenants: 1
users_per_tenant: 1
sla:
failure_rate:
max: 0