openstack/rally
Code Issues Proposed changes

OSProfiler Intergration

Browse Source 
1. connect_string store in deployment and get in credential
2. if has got connect_string and hmac_key,rally get osprofiler data
   and process them in osprofilerchart
3. new js code will receive the data from osprofiler chart and
   generate graphs

Change-Id: Id477a10489102425fae79ecf6719b1e0739e92c0
This commit is contained in:
cxhuawei 2017-09-03 22:55:35 -07:00 committed by Andrey Kurilin
parent 2c881858f9
commit 37901c7893
17 changed files with 219 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
rally/cli/envutils.py
View File

@ -108,7 +108,8 @@ def get_creds_from_env_vars():
"https_cacert": os.environ.get("OS_CACERT", ""),
"https_insecure": strutils.bool_from_string(
os.environ.get("OS_INSECURE")),
"profiler_hmac_key": os.environ.get("OSPROFILER_HMAC_KEY")
"profiler_hmac_key": os.environ.get("OSPROFILER_HMAC_KEY"),
"profiler_conn_str": os.environ.get("OSPROFILER_CONN_STR")
}
user_domain_name = os.environ.get("OS_USER_DOMAIN_NAME")

1
rally/deployment/engines/existing.py
View File

@ -153,6 +153,7 @@ class ExistingCloud(engine.Engine):
"https_insecure": {"type": "boolean"},
"https_cacert": {"type": "string"},
"profiler_hmac_key": {"type": ["string", "null"]},
"profiler_conn_str": {"type": ["string", "null"]},
"admin": USER_SCHEMA,
"users": {"type": "array", "items": USER_SCHEMA, "minItems": 1},
"extra": {"type": "object", "additionalProperties": True}

3
rally/plugins/openstack/context/keystone/users.py
View File

@ -217,7 +217,8 @@ class UserGenerator(context.Context):
https_insecure=self.credential.https_insecure,
https_cacert=self.credential.https_cacert,
region_name=self.credential.region_name,
profiler_hmac_key=self.credential.profiler_hmac_key)
profiler_hmac_key=self.credential.profiler_hmac_key,
profiler_conn_str=self.credential.profiler_conn_str)
users.append({"id": user.id,
"credential": user_credential,
"tenant_id": tenant_id})

12
rally/plugins/openstack/credential.py
View File

@ -32,7 +32,7 @@ class OpenStackCredential(credential.Credential):
domain_name=None, endpoint=None, user_domain_name=None,
project_domain_name=None,
https_insecure=False, https_cacert=None,
profiler_hmac_key=None):
profiler_hmac_key=None, profiler_conn_str=None):
self.auth_url = auth_url
self.username = username
self.password = password
@ -47,6 +47,7 @@ class OpenStackCredential(credential.Credential):
self.https_insecure = https_insecure
self.https_cacert = https_cacert
self.profiler_hmac_key = profiler_hmac_key
self.profiler_conn_str = profiler_conn_str
self._clients_cache = {}
@ -78,7 +79,8 @@ class OpenStackCredential(credential.Credential):
"user_domain_name": self.user_domain_name,
"project_domain_name": self.project_domain_name,
"permission": self.permission,
"profiler_hmac_key": self.profiler_hmac_key}
"profiler_hmac_key": self.profiler_hmac_key,
"profiler_conn_str": self.profiler_conn_str}
def verify_connection(self):
if self.permission == consts.EndpointPermission.ADMIN:
@ -150,7 +152,8 @@ class OpenStackCredentialBuilder(credential.CredentialBuilder):
None]},
"https_insecure": {"type": "boolean"},
"https_cacert": {"type": "string"},
"profiler_hmac_key": {"type": ["string", "null"]}
"profiler_hmac_key": {"type": ["string", "null"]},
"profiler_conn_str": {"type": ["string", "null"]}
},
"anyOf": [
{"description": "The case when the admin is specified and the "
@ -179,7 +182,8 @@ class OpenStackCredentialBuilder(credential.CredentialBuilder):
project_domain_name=user.get("project_domain_name", None),
https_insecure=common.get("https_insecure", False),
https_cacert=common.get("https_cacert"),
profiler_hmac_key=common.get("profiler_hmac_key"))
profiler_hmac_key=common.get("profiler_hmac_key"),
profiler_conn_str=common.get("profiler_conn_str"))
return cred.to_dict()
def build_credentials(self):

0
rally/plugins/openstack/embedcharts/__init__.py Normal file
View File

88
rally/plugins/openstack/embedcharts/osprofilerchart.py Normal file
View File

@ -0,0 +1,88 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
from rally.common import logging
from rally.common.plugin import plugin
from rally.task.processing.charts import OutputTextArea
LOG = logging.getLogger(__name__)
def _datetime_json_serialize(obj):
if hasattr(obj, "isoformat"):
return obj.isoformat()
else:
return obj
@plugin.configure(name="OSProfiler")
class OSProfilerChart(OutputTextArea):
"""osprofiler content
This plugin complete data of osprofiler
"""
widget = "OSProfiler"
@classmethod
def get_osprofiler_data(cls, data):
from osprofiler import cmd
from osprofiler.drivers import base
try:
engine = base.get_driver(data["data"]["conn_str"])
except Exception:
if logging.is_debug():
LOG.exception("Error while fetching OSProfiler results.")
return None
data["widget"] = "EmbedChart"
data["title"] = "{0} : {1}".format(data["title"],
data["data"]["trace_id"][0])
path = "%s/template.html" % os.path.dirname(cmd.__file__)
with open(path) as f:
html_obj = f.read()
osp_data = engine.get_report(data["data"]["trace_id"][0])
osp_data = json.dumps(osp_data,
indent=4,
separators=(",", ": "),
default=_datetime_json_serialize)
data["data"] = html_obj.replace("$DATA", osp_data)
data["data"] = data["data"].replace("$LOCAL", "false")
# NOTE(chenxu): self._data will be passed to
# ["complete_output"]["data"] as a whole string and
# tag </script> will be parsed incorrectly in javascript string
# so we turn it to <\/script> and turn it back in javascript.
data["data"] = data["data"].replace("/script>", "\/script>")
return {"title": data["title"],
"widget": data["widget"],
"data": data["data"]}
@classmethod
def render_complete_data(cls, data):
if data["data"].get("conn_str"):
result = cls.get_osprofiler_data(data)
if result:
return result
return {"title": data["title"],
"widget": "TextArea",
"data": data["data"]["trace_id"]}

12
rally/plugins/openstack/scenario.py
View File

@ -121,19 +121,23 @@ class OpenStackScenario(scenario.Scenario):
if context is not None:
cred = None
profiler_hmac_key = None
profiler_conn_str = None
if context.get("admin"):
cred = context["admin"]["credential"]
if cred.profiler_hmac_key is not None:
profiler_hmac_key = cred.profiler_hmac_key
profiler_conn_str = cred.profiler_conn_str
if context.get("user"):
cred = context["user"]["credential"]
if cred.profiler_hmac_key is not None:
profiler_hmac_key = cred.profiler_hmac_key
profiler_conn_str = cred.profiler_conn_str
if profiler_hmac_key is None:
return
profiler.init(profiler_hmac_key)
trace_id = profiler.get().get_base_id()
self.add_output(complete={
"title": "OSProfiler Trace-ID",
"chart_plugin": "TextArea",
"data": [trace_id]})
complete_data = {"title": "OSProfiler Trace-ID",
"chart_plugin": "OSProfiler",
"data": {"trace_id": [trace_id],
"conn_str": profiler_conn_str}}
self.add_output(complete=complete_data)

1
rally/task/processing/charts.py
View File

@ -16,7 +16,6 @@ import abc
import bisect
import collections
import math
import six
from rally.common.plugin import plugin

22
rally/ui/templates/task/directive_widget.js
View File

@ -1,3 +1,11 @@
function changeFrameHeight(){
var ifm= document.getElementById("embedchart");
ifm.height=document.documentElement.clientHeight;
ifm.width=document.documentElement.clientWidth;
}
window.onresize=function(){
changeFrameHeight();
}
var widgetDirective = function($compile) {
var Chart = {
_render: function(node, data, chart, do_after){
@ -141,8 +149,20 @@ var widgetDirective = function($compile) {
} else if (attrs.widget === "TextArea") {
var template = "<div style='padding:0 0 5px' ng-repeat='str in data track by $index'>{{str}}</div><div style='height:10px'></div>";
var el = element.empty().append($compile(template)(scope)).children()[0]
} else {
}
else if (attrs.widget == "EmbedChart") {
/* NOTE(chenxu): tag <\/script> in javascript string will be parsed incorrectly.
so we convert <\/script> to <\\/script> in python and convert it back here. */
data = data.replace(/\\\/script>/ig, "\/script>");
var template = "<iframe scrolling='no' id='embedchart' frameborder='0' onload='changeFrameHeight()' style='width:100%;'></iframe>"
var el = element.empty().append($compile(template)(scope)).children()[0];
var iframe = el.contentWindow || ( el.contentDocument.document || el.contentDocument);
iframe.document.open();
iframe.document.write(data);
iframe.document.close();
}
else {
var el_chart = element.addClass("chart").css({display:"block"});
var el = el_chart.html("<svg></svg>").children()[0];

3
samples/deployments/existing-keystone-v3-osprofiler.json
View File

@ -12,6 +12,7 @@
},
"https_insecure": false,
"https_cacert": "",
"profiler_hmac_key": "SECRET_KEY"
"profiler_hmac_key": "SECRET_KEY",
"profiler_conn_str": "mongodb://localhost"
}
}

12
tests/unit/cli/commands/test_deployment.py
View File

@ -62,7 +62,8 @@ class DeploymentCommandsTestCase(test.TestCase):
"OS_INSECURE": "True",
"OS_CACERT": "fake_cacert",
"RALLY_DEPLOYMENT": "fake_deployment_id",
"OSPROFILER_HMAC_KEY": "fake_hmac_key"})
"OSPROFILER_HMAC_KEY": "fake_hmac_key",
"OSPROFILER_CONN_STR": "fake_conn_str"})
@mock.patch("rally.cli.commands.deployment.DeploymentCommands.list")
def test_createfromenv_keystonev2(self, mock_list):
self.deployment.create(self.fake_api, "from_env", True)
@ -80,7 +81,8 @@ class DeploymentCommandsTestCase(test.TestCase):
},
"https_insecure": True,
"https_cacert": "fake_cacert",
"profiler_hmac_key": "fake_hmac_key"
"profiler_hmac_key": "fake_hmac_key",
"profiler_conn_str": "fake_conn_str"
}
},
name="from_env"
@ -98,7 +100,8 @@ class DeploymentCommandsTestCase(test.TestCase):
"OS_INSECURE": "True",
"OS_CACERT": "fake_cacert",
"RALLY_DEPLOYMENT": "fake_deployment_id",
"OSPROFILER_HMAC_KEY": "fake_hmac_key"})
"OSPROFILER_HMAC_KEY": "fake_hmac_key",
"OSPROFILER_CONN_STR": "fake_conn_str"})
@mock.patch("rally.cli.commands.deployment.DeploymentCommands.list")
def test_createfromenv_keystonev3(self, mock_list):
self.deployment.create(self.fake_api, "from_env", True)
@ -118,7 +121,8 @@ class DeploymentCommandsTestCase(test.TestCase):
},
"https_insecure": True,
"https_cacert": "fake_cacert",
"profiler_hmac_key": "fake_hmac_key"
"profiler_hmac_key": "fake_hmac_key",
"profiler_conn_str": "fake_conn_str"
}
},
name="from_env"

9
tests/unit/cli/test_envutils.py
View File

@ -113,6 +113,7 @@ class EnvUtilsTestCase(test.TestCase):
"OS_ENDPOINT": "fake_endpoint",
"OS_INSECURE": "True",
"OSPROFILER_HMAC_KEY": "fake_hmac_key",
"OSPROFILER_CONN_STR": "fake_conn_str",
"OS_CACERT": "fake_cacert"})
def test_get_creds_from_env_vars_keystone_v2(self):
expected_creds = {
@ -127,7 +128,8 @@ class EnvUtilsTestCase(test.TestCase):
"region_name": "fake_region_name",
"https_cacert": "fake_cacert",
"https_insecure": True,
"profiler_hmac_key": "fake_hmac_key"
"profiler_hmac_key": "fake_hmac_key",
"profiler_conn_str": "fake_conn_str"
}
creds = envutils.get_creds_from_env_vars()
self.assertEqual(expected_creds, creds)
@ -143,6 +145,7 @@ class EnvUtilsTestCase(test.TestCase):
"OS_PROJECT_DOMAIN_NAME": "fake_pdn",
"OS_USER_DOMAIN_NAME": "fake_udn",
"OSPROFILER_HMAC_KEY": "fake_hmac_key",
"OSPROFILER_CONN_STR": "fake_conn_str",
"OS_CACERT": "fake_cacert"})
def test_get_creds_from_env_vars_keystone_v3(self):
expected_creds = {
@ -159,7 +162,8 @@ class EnvUtilsTestCase(test.TestCase):
"region_name": "fake_region_name",
"https_cacert": "fake_cacert",
"https_insecure": True,
"profiler_hmac_key": "fake_hmac_key"
"profiler_hmac_key": "fake_hmac_key",
"profiler_conn_str": "fake_conn_str"
}
creds = envutils.get_creds_from_env_vars()
self.assertEqual(expected_creds, creds)
@ -170,6 +174,7 @@ class EnvUtilsTestCase(test.TestCase):
"OS_ENDPOINT": "fake_endpoint",
"OS_INSECURE": "True",
"OSPROFILER_HMAC_KEY": "fake_hmac_key",
"OSPROFILER_CONN_STR": "fake_conn_str",
"OS_CACERT": "fake_cacert"})
def test_get_creds_from_env_vars_when_required_vars_missing(self):
if "OS_USERNAME" in os.environ:

9
tests/unit/common/objects/test_credential.py
View File

@ -40,7 +40,8 @@ class CredentialTestCase(test.TestCase):
"https_cacert": None,
"project_domain_name": None,
"user_domain_name": None,
"profiler_hmac_key": None})
"profiler_hmac_key": None,
"profiler_conn_str": None})
def test_to_dict_with_include_permission(self):
credential = objects.Credential(
@ -61,7 +62,8 @@ class CredentialTestCase(test.TestCase):
"https_cacert": None,
"project_domain_name": None,
"user_domain_name": None,
"profiler_hmac_key": None})
"profiler_hmac_key": None,
"profiler_conn_str": None})
def test_to_dict_with_kwarg_credential(self):
credential = objects.Credential(
@ -83,4 +85,5 @@ class CredentialTestCase(test.TestCase):
"https_cacert": None,
"project_domain_name": None,
"user_domain_name": None,
"profiler_hmac_key": None})
"profiler_hmac_key": None,
"profiler_conn_str": None})

4
tests/unit/deployment/engines/test_existing.py
View File

@ -38,6 +38,7 @@ class TestExistingCloud(test.TestCase):
"https_insecure": False,
"https_cacert": "cacert",
"profiler_hmac_key": None,
"profiler_conn_str": None,
"admin": {
"username": "admin",
"password": "myadminpass",
@ -54,6 +55,7 @@ class TestExistingCloud(test.TestCase):
"https_insecure": False,
"https_cacert": "cacert",
"profiler_hmac_key": None,
"profiler_conn_str": None,
"admin": {
"username": "admin",
"password": "myadminpass",
@ -75,6 +77,7 @@ class TestExistingCloud(test.TestCase):
"https_insecure": False,
"https_cacert": "cacert",
"profiler_hmac_key": None,
"profiler_conn_str": None,
"admin": {
"username": "admin",
"password": "myadminpass",
@ -163,6 +166,7 @@ class TestExistingCloud(test.TestCase):
"https_cacert": "cacert",
"https_insecure": False,
"profiler_hmac_key": None,
"profiler_conn_str": None,
"password": "myadminpass",
"permission": "admin",
"project_domain_name": None,

56
tests/unit/plugins/openstack/embedcharts/test_osprofilerchart.py Normal file
View File

@ -0,0 +1,56 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from rally.plugins.openstack.embedcharts.osprofilerchart import OSProfilerChart
from tests.unit import test
class OSProfilerChartTestCase(test.TestCase):
class OSProfilerChart(OSProfilerChart):
widget = "OSProfiler"
@mock.patch("osprofiler.drivers.base.get_driver")
def test_get_osprofiler_data(self, mock_get_driver):
engine = mock.Mock()
attrs = {"get_report.return_value": "html"}
engine.configure_mock(**attrs)
mock_get_driver.return_value = engine
data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
return_data = OSProfilerChart.render_complete_data(data)
self.assertEqual("EmbedChart", return_data["widget"])
self.assertEqual("a : 1", return_data["title"])
data = {"data": {"conn_str": None, "trace_id": ["1"]}, "title": "a"}
return_data = OSProfilerChart.render_complete_data(data)
self.assertEqual("TextArea", return_data["widget"])
self.assertEqual(["1"], return_data["data"])
self.assertEqual("a", return_data["title"])
mock_get_driver.side_effect = Exception
data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
return_data = OSProfilerChart.render_complete_data(data)
self.assertEqual("TextArea", return_data["widget"])
self.assertEqual(["1"], return_data["data"])
self.assertEqual("a", return_data["title"])
def test_datetime_json_serialize(self):
from rally.plugins.openstack.embedcharts.osprofilerchart \
import _datetime_json_serialize
A = mock.Mock()
B = A.isoformat()
self.assertEqual(B, _datetime_json_serialize(A))
self.assertEqual("C", _datetime_json_serialize("C"))

5
tests/unit/plugins/openstack/test_credential.py
View File

@ -45,7 +45,8 @@ class OpenStackCredentialTestCase(test.TestCase):
"https_cacert": None,
"project_domain_name": None,
"user_domain_name": None,
"profiler_hmac_key": None},
"profiler_hmac_key": None,
"profiler_conn_str": None},
self.credential.to_dict())
@mock.patch("rally.plugins.openstack.osclients.Clients")
@ -130,6 +131,7 @@ class OpenStackCredentialBuilderTestCase(test.TestCase):
"https_cacert": "cacert",
"https_insecure": False,
"profiler_hmac_key": None,
"profiler_conn_str": None,
"project_domain_name": None,
"region_name": "RegionOne",
"tenant_name": "demo",
@ -147,6 +149,7 @@ class OpenStackCredentialBuilderTestCase(test.TestCase):
"https_cacert": "cacert",
"https_insecure": False,
"profiler_hmac_key": None,
"profiler_conn_str": None,
"project_domain_name": None,
"region_name": "RegionOne",
"tenant_name": "demo",

3
tests/unit/test_api.py
View File

@ -43,7 +43,8 @@ FAKE_DEPLOYMENT_CONFIG = {
"domain_name": None,
"project_domain_name": "Default",
"user_domain_name": "Default",
"profiler_hmac_key": None
"profiler_hmac_key": None,
"profiler_conn_str": None
},
"region_name": "RegionOne",
"endpoint_type": consts.EndpointType.INTERNAL}}