OSProfiler Intergration

1. connect_string store in deployment and get in credential
2. if has got connect_string and hmac_key,rally get osprofiler data
   and process them in osprofilerchart
3. new js code will receive the data from osprofiler chart and
   generate graphs

Change-Id: Id477a10489102425fae79ecf6719b1e0739e92c0

rally/plugins/openstack/context/keystone/users.py

@@ -217,7 +217,8 @@ class UserGenerator(context.Context):
                 https_insecure=self.credential.https_insecure,
                 https_cacert=self.credential.https_cacert,
                 region_name=self.credential.region_name,
-                profiler_hmac_key=self.credential.profiler_hmac_key)
+                profiler_hmac_key=self.credential.profiler_hmac_key,
+                profiler_conn_str=self.credential.profiler_conn_str)
             users.append({"id": user.id,
                           "credential": user_credential,
                           "tenant_id": tenant_id})

rally/plugins/openstack/credential.py

@@ -32,7 +32,7 @@ class OpenStackCredential(credential.Credential):
                  domain_name=None, endpoint=None, user_domain_name=None,
                  project_domain_name=None,
                  https_insecure=False, https_cacert=None,
-                 profiler_hmac_key=None):
+                 profiler_hmac_key=None, profiler_conn_str=None):
         self.auth_url = auth_url
         self.username = username
         self.password = password
@@ -47,6 +47,7 @@ class OpenStackCredential(credential.Credential):
         self.https_insecure = https_insecure
         self.https_cacert = https_cacert
         self.profiler_hmac_key = profiler_hmac_key
+        self.profiler_conn_str = profiler_conn_str
 
         self._clients_cache = {}
 
@@ -78,7 +79,8 @@ class OpenStackCredential(credential.Credential):
                 "user_domain_name": self.user_domain_name,
                 "project_domain_name": self.project_domain_name,
                 "permission": self.permission,
-                "profiler_hmac_key": self.profiler_hmac_key}
+                "profiler_hmac_key": self.profiler_hmac_key,
+                "profiler_conn_str": self.profiler_conn_str}
 
     def verify_connection(self):
         if self.permission == consts.EndpointPermission.ADMIN:
@@ -150,7 +152,8 @@ class OpenStackCredentialBuilder(credential.CredentialBuilder):
                          None]},
             "https_insecure": {"type": "boolean"},
             "https_cacert": {"type": "string"},
-            "profiler_hmac_key": {"type": ["string", "null"]}
+            "profiler_hmac_key": {"type": ["string", "null"]},
+            "profiler_conn_str": {"type": ["string", "null"]}
         },
         "anyOf": [
             {"description": "The case when the admin is specified and the "
@@ -179,7 +182,8 @@ class OpenStackCredentialBuilder(credential.CredentialBuilder):
             project_domain_name=user.get("project_domain_name", None),
             https_insecure=common.get("https_insecure", False),
             https_cacert=common.get("https_cacert"),
-            profiler_hmac_key=common.get("profiler_hmac_key"))
+            profiler_hmac_key=common.get("profiler_hmac_key"),
+            profiler_conn_str=common.get("profiler_conn_str"))
         return cred.to_dict()
 
     def build_credentials(self):

rally/plugins/openstack/embedcharts/osprofilerchart.py

@@ -0,0 +1,88 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import os
+from rally.common import logging
+from rally.common.plugin import plugin
+from rally.task.processing.charts import OutputTextArea
+
+LOG = logging.getLogger(__name__)
+
+
+def _datetime_json_serialize(obj):
+    if hasattr(obj, "isoformat"):
+        return obj.isoformat()
+    else:
+        return obj
+
+
+@plugin.configure(name="OSProfiler")
+class OSProfilerChart(OutputTextArea):
+    """osprofiler content
+
+    This plugin complete data of osprofiler
+
+    """
+
+    widget = "OSProfiler"
+
+    @classmethod
+    def get_osprofiler_data(cls, data):
+
+        from osprofiler import cmd
+        from osprofiler.drivers import base
+
+        try:
+            engine = base.get_driver(data["data"]["conn_str"])
+        except Exception:
+            if logging.is_debug():
+                LOG.exception("Error while fetching OSProfiler results.")
+            return None
+
+        data["widget"] = "EmbedChart"
+        data["title"] = "{0} : {1}".format(data["title"],
+                                           data["data"]["trace_id"][0])
+
+        path = "%s/template.html" % os.path.dirname(cmd.__file__)
+        with open(path) as f:
+            html_obj = f.read()
+
+        osp_data = engine.get_report(data["data"]["trace_id"][0])
+        osp_data = json.dumps(osp_data,
+                              indent=4,
+                              separators=(",", ": "),
+                              default=_datetime_json_serialize)
+        data["data"] = html_obj.replace("$DATA", osp_data)
+        data["data"] = data["data"].replace("$LOCAL", "false")
+
+        # NOTE(chenxu): self._data will be passed to
+        # ["complete_output"]["data"] as a whole string and
+        # tag </script> will be parsed incorrectly in javascript string
+        # so we turn it to <\/script> and turn it back in javascript.
+        data["data"] = data["data"].replace("/script>", "\/script>")
+
+        return {"title": data["title"],
+                "widget": data["widget"],
+                "data": data["data"]}
+
+    @classmethod
+    def render_complete_data(cls, data):
+        if data["data"].get("conn_str"):
+            result = cls.get_osprofiler_data(data)
+            if result:
+                return result
+        return {"title": data["title"],
+                "widget": "TextArea",
+                "data": data["data"]["trace_id"]}

rally/plugins/openstack/scenario.py

@@ -121,19 +121,23 @@ class OpenStackScenario(scenario.Scenario):
         if context is not None:
             cred = None
             profiler_hmac_key = None
+            profiler_conn_str = None
             if context.get("admin"):
                 cred = context["admin"]["credential"]
                 if cred.profiler_hmac_key is not None:
                     profiler_hmac_key = cred.profiler_hmac_key
+                    profiler_conn_str = cred.profiler_conn_str
             if context.get("user"):
                 cred = context["user"]["credential"]
                 if cred.profiler_hmac_key is not None:
                     profiler_hmac_key = cred.profiler_hmac_key
+                    profiler_conn_str = cred.profiler_conn_str
             if profiler_hmac_key is None:
                 return
             profiler.init(profiler_hmac_key)
             trace_id = profiler.get().get_base_id()
-            self.add_output(complete={
-                "title": "OSProfiler Trace-ID",
-                "chart_plugin": "TextArea",
-                "data": [trace_id]})
+            complete_data = {"title": "OSProfiler Trace-ID",
+                             "chart_plugin": "OSProfiler",
+                             "data": {"trace_id": [trace_id],
+                                      "conn_str": profiler_conn_str}}
+            self.add_output(complete=complete_data)

samples/deployments/existing-keystone-v3-osprofiler.json

@@ -12,6 +12,7 @@
         },
         "https_insecure": false,
         "https_cacert": "",
-        "profiler_hmac_key": "SECRET_KEY"
+        "profiler_hmac_key": "SECRET_KEY",
+        "profiler_conn_str": "mongodb://localhost"
     }
 }

tests/unit/plugins/openstack/embedcharts/test_osprofilerchart.py

@@ -0,0 +1,56 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+from rally.plugins.openstack.embedcharts.osprofilerchart import OSProfilerChart
+from tests.unit import test
+
+
+class OSProfilerChartTestCase(test.TestCase):
+
+    class OSProfilerChart(OSProfilerChart):
+        widget = "OSProfiler"
+
+    @mock.patch("osprofiler.drivers.base.get_driver")
+    def test_get_osprofiler_data(self, mock_get_driver):
+        engine = mock.Mock()
+        attrs = {"get_report.return_value": "html"}
+        engine.configure_mock(**attrs)
+        mock_get_driver.return_value = engine
+
+        data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
+        return_data = OSProfilerChart.render_complete_data(data)
+        self.assertEqual("EmbedChart", return_data["widget"])
+        self.assertEqual("a : 1", return_data["title"])
+
+        data = {"data": {"conn_str": None, "trace_id": ["1"]}, "title": "a"}
+        return_data = OSProfilerChart.render_complete_data(data)
+        self.assertEqual("TextArea", return_data["widget"])
+        self.assertEqual(["1"], return_data["data"])
+        self.assertEqual("a", return_data["title"])
+
+        mock_get_driver.side_effect = Exception
+        data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
+        return_data = OSProfilerChart.render_complete_data(data)
+        self.assertEqual("TextArea", return_data["widget"])
+        self.assertEqual(["1"], return_data["data"])
+        self.assertEqual("a", return_data["title"])
+
+    def test_datetime_json_serialize(self):
+        from rally.plugins.openstack.embedcharts.osprofilerchart \
+            import _datetime_json_serialize
+        A = mock.Mock()
+        B = A.isoformat()
+        self.assertEqual(B, _datetime_json_serialize(A))
+        self.assertEqual("C", _datetime_json_serialize("C"))

tests/unit/plugins/openstack/test_credential.py

@@ -45,7 +45,8 @@ class OpenStackCredentialTestCase(test.TestCase):
                           "https_cacert": None,
                           "project_domain_name": None,
                           "user_domain_name": None,
-                          "profiler_hmac_key": None},
+                          "profiler_hmac_key": None,
+                          "profiler_conn_str": None},
                          self.credential.to_dict())
 
     @mock.patch("rally.plugins.openstack.osclients.Clients")
@@ -130,6 +131,7 @@ class OpenStackCredentialBuilderTestCase(test.TestCase):
                 "https_cacert": "cacert",
                 "https_insecure": False,
                 "profiler_hmac_key": None,
+                "profiler_conn_str": None,
                 "project_domain_name": None,
                 "region_name": "RegionOne",
                 "tenant_name": "demo",
@@ -147,6 +149,7 @@ class OpenStackCredentialBuilderTestCase(test.TestCase):
                     "https_cacert": "cacert",
                     "https_insecure": False,
                     "profiler_hmac_key": None,
+                    "profiler_conn_str": None,
                     "project_domain_name": None,
                     "region_name": "RegionOne",
                     "tenant_name": "demo",