Implement storing osprofiler reports separately from rally report

Change-Id: I8c95e7f433a6f22ae818a78ede4b12bb84cf1f5d

devstack/lib/rally

@@ -51,7 +51,9 @@ OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-""}
 #       OSPROFILER_HMAC_KEYS rally html report will use osprofiler api to
 #       generate html report for each trace and embed it as iframe to our
 #       native html repor
+#   ``RALLY_OSPROFILER_CHART`` - optional, a path to store osprofiler's reports
 #
+
 # _create_deployment_config filename
 function _create_deployment_config() {
 if [[ "$IDENTITY_API_VERSION" == 2.0 ]]

rally_openstack/cfg/opts.py

@@ -40,6 +40,8 @@ from rally_openstack.cfg import keystone_users
 
 from rally_openstack.cfg import cleanup
 
+from rally_openstack.embedcharts import osprofilerchart
+
 
 def list_opts():
 
@@ -49,7 +51,8 @@ def list_opts():
                    nova.OPTS, osclients.OPTS, profiler.OPTS, sahara.OPTS,
                    vm.OPTS, glance.OPTS, watcher.OPTS, tempest.OPTS,
                    keystone_roles.OPTS, keystone_users.OPTS, cleanup.OPTS,
-                   senlin.OPTS, neutron.OPTS, octavia.OPTS):
+                   senlin.OPTS, neutron.OPTS, octavia.OPTS,
+                   osprofilerchart.OPTS):
         for category, opt in l_opts.items():
             opts.setdefault(category, [])
             opts[category].extend(opt)

rally_openstack/embedcharts/osprofilerchart.py

@@ -15,12 +15,42 @@
 import json
 import os
 
+from rally.common import cfg
 from rally.common import logging
 from rally.common import opts
 from rally.common.plugin import plugin
-from rally.task.processing.charts import OutputTextArea
+from rally.task.processing import charts
+
+import rally_openstack
+
+
+if rally_openstack.__rally_version__ < (1, 5, 0):
+    # NOTE(andreykurilin): this is a workaround to make inheritance of
+    #   OSProfilerChart clear.
+    OutputEmbeddedChart = type("OutputEmbeddedChart", (object, ), {})
+    OutputEmbeddedExternalChart = type("OutputEmbeddedExternalChart",
+                                       (object, ), {})
+else:
+    OutputEmbeddedChart = charts.OutputEmbeddedChart
+    OutputEmbeddedExternalChart = charts.OutputEmbeddedExternalChart
+
+
+OPTS = {
+    "openstack": [
+        cfg.StrOpt(
+            "osprofiler_chart_mode",
+            default=None,
+            help="Mode of embedding OSProfiler's chart. Can be 'text' "
+                 "(embed only trace id), 'raw' (embed raw osprofiler's native "
+                 "report) or a path to directory (raw osprofiler's native "
+                 "reports for each iteration will be saved separately there "
+                 "to decrease the size of rally report itself)")
+    ]
+}
+
 
 LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
 
 
 def _datetime_json_serialize(obj):
@@ -31,25 +61,20 @@ def _datetime_json_serialize(obj):
 
 
 @plugin.configure(name="OSProfiler")
-class OSProfilerChart(OutputTextArea):
+class OSProfilerChart(OutputEmbeddedChart,
-    """OSProfiler content
+                      OutputEmbeddedExternalChart,
-
+                      charts.OutputTextArea):
-    This plugin complete data of osprofiler
+    """Chart for embedding OSProfiler data."""
-    """
-
-    widget = "OSProfiler"
 
     @classmethod
-    def get_osprofiler_data(cls, data):
+    def _fetch_osprofiler_data(cls, connection_str, trace_id):
-
-        from osprofiler import cmd
         from osprofiler.drivers import base
         from osprofiler import opts as osprofiler_opts
 
         opts.register_opts(osprofiler_opts.list_opts())
 
         try:
-            engine = base.get_driver(data["data"]["conn_str"])
+            engine = base.get_driver(connection_str)
         except Exception:
             msg = "Error while fetching OSProfiler results."
             if logging.is_debug():
@@ -58,38 +83,80 @@ class OSProfilerChart(OutputTextArea):
                 LOG.error(msg)
             return None
 
-        data["widget"] = "EmbedChart"
+        return engine.get_report(trace_id)
-        data["title"] = "{0} : {1}".format(data["title"],
+
-                                           data["data"]["trace_id"][0])
+    @classmethod
+    def _generate_osprofiler_report(cls, osp_data):
+        from osprofiler import cmd
 
         path = "%s/template.html" % os.path.dirname(cmd.__file__)
         with open(path) as f:
             html_obj = f.read()
 
-        osp_data = engine.get_report(data["data"]["trace_id"][0])
         osp_data = json.dumps(osp_data,
                               indent=4,
                               separators=(",", ": "),
                               default=_datetime_json_serialize)
-        data["data"] = html_obj.replace("$DATA", osp_data)
+        return html_obj.replace("$DATA", osp_data).replace("$LOCAL", "false")
-        data["data"] = data["data"].replace("$LOCAL", "false")
 
-        # NOTE(chenxu): self._data will be passed to
+    @classmethod
-        # ["complete_output"]["data"] as a whole string and
+    def _return_raw_response_for_complete_data(cls, data):
-        # tag </script> will be parsed incorrectly in javascript string
+        return charts.OutputTextArea.render_complete_data({
-        # so we turn it to <\/script> and turn it back in javascript.
+            "title": data["title"],
-        data["data"] = data["data"].replace("/script>", "\/script>")
+            "widget": "TextArea",
-
+            "data": [data["data"]["trace_id"]]
-        return {"title": data["title"],
+        })
-                "widget": data["widget"],
-                "data": data["data"]}
 
     @classmethod
     def render_complete_data(cls, data):
-        if data["data"].get("conn_str"):
+        mode = CONF.openstack.osprofiler_chart_mode
-            result = cls.get_osprofiler_data(data)
+
-            if result:
+        if isinstance(data["data"]["trace_id"], list):
-                return result
+            # NOTE(andreykurilin): it is an adoption for the format that we
-        return {"title": data["title"],
+            #   used  before rally-openstack 1.5.0 .
-                "widget": "TextArea",
+            data["data"]["trace_id"] = data["data"]["trace_id"][0]
-                "data": data["data"]["trace_id"]}
+
+        if data["data"].get("conn_str") and mode != "text":
+            osp_data = cls._fetch_osprofiler_data(
+                data["data"]["conn_str"],
+                trace_id=data["data"]["trace_id"]
+            )
+            if not osp_data:
+                # for some reasons we failed to fetch data from OSProfiler's
+                # backend. in this case we can display just trace ID
+                return cls._return_raw_response_for_complete_data(data)
+
+            osp_report = cls._generate_osprofiler_report(osp_data)
+            title = "{0} : {1}".format(data["title"],
+                                       data["data"]["trace_id"])
+
+            if rally_openstack.__rally_version__ < (1, 5, 0):
+                return {
+                    "title": title,
+                    "widget": "EmbeddedChart",
+                    "data": osp_report.replace("/script>", "\\/script>")
+                }
+            elif (mode and mode != "raw") and "workload_uuid" in data["data"]:
+                # NOTE(andreykurilin): we need to rework our charts plugin
+                #   mechanism so it is available out of box
+                workload_uuid = data["data"]["workload_uuid"]
+                iteration = data["data"]["iteration"]
+                file_name = "w_%s-%s.html" % (workload_uuid, iteration)
+                path = os.path.join(mode, file_name)
+                with open(path, "w") as f:
+                    f.write(osp_report)
+                return OutputEmbeddedExternalChart.render_complete_data(
+                    {
+                        "title": title,
+                        "widget": "EmbeddedChart",
+                        "data": path
+                    }
+                )
+            else:
+                return OutputEmbeddedChart.render_complete_data(
+                    {"title": title,
+                     "widget": "EmbeddedChart",
+                     "data": osp_report}
+                )
+
+        return cls._return_raw_response_for_complete_data(data)

rally_openstack/scenario.py

@@ -112,7 +112,9 @@ class OpenStackScenario(scenario.Scenario):
         if not CONF.openstack.enable_profiler:
             return
 
-        if context is not None:
+        # False statement here means that Scenario class is used outside the
+        # runner as some kind of utils
+        if context is not None and "iteration" in context:
 
             profiler_hmac_key = None
             profiler_conn_str = None
@@ -132,6 +134,9 @@ class OpenStackScenario(scenario.Scenario):
             trace_id = profiler.get().get_base_id()
             complete_data = {"title": "OSProfiler Trace-ID",
                              "chart_plugin": "OSProfiler",
-                             "data": {"trace_id": [trace_id],
+                             "data": {"trace_id": trace_id,
-                                      "conn_str": profiler_conn_str}}
+                                      "conn_str": profiler_conn_str,
+                                      "taskID": context["task"]["uuid"],
+                                      "workload_uuid": context["owner_id"],
+                                      "iteration": context["iteration"]}}
             self.add_output(complete=complete_data)

tests/ci/playbooks/roles/prepare-for-rally-task/defaults/main.yaml

@@ -3,4 +3,5 @@ existing_user_password_1: "rally-test-password-1"
 existing_user_project_1: "rally-test-project-1"
 existing_user_name_2: "rally-test-user-2"
 existing_user_password_2: "rally-test-password-2"
-existing_user_project_2: "rally-test-project-2"
+existing_user_project_2: "rally-test-project-2"
+RALLY_OSPROFILER_CHART: "osprofiler_reports"

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml

@@ -16,6 +16,24 @@
     owner: stack
     group: stack
 
+- name: Create directory for OSProfiler reports
+  become: True
+  become_user: stack
+  file:
+    path: '{{ rally_home_dir }}/results/{{ RALLY_OSPROFILER_CHART }}'
+    state: directory
+    owner: stack
+    group: stack
+
+- name: Extend Rally config with
+  become: True
+  become_user: stack
+  shell:
+    executable: /bin/bash
+    cmd: |
+      echo "[openstack]" >> /etc/rally/rally.conf
+      echo "osprofiler_chart_mode={{ RALLY_OSPROFILER_CHART }}" >> /etc/rally/rally.conf
+
 - name: Create a directory for custom plugins
   become: True
   become_user: stack
@@ -145,16 +163,16 @@
   shell: rally env create --name devstask-with-users --spec "{{ rally_existing_users_config }}"
   when: rally_use_existing_users == True
 
-- name: Print Rally deployment config
-  become: True
-  become_user: stack
-  command: "rally deployment config"
-
 - name: Check Environment works
   become: True
   become_user: stack
   command: "rally --debug env check"
 
+- name: Print Rally deployment config
+  become: True
+  become_user: stack
+  command: "rally deployment config"
+
 - name: Print Environment info
   become: True
   become_user: stack

tests/ci/playbooks/roles/process-task-results/tasks/main.yaml

@@ -1,7 +1,12 @@
 - name: Generate a HTML report
   become: True
   become_user: stack
-  command: rally task report --html-static --out {{ rally_results_dir }}/report.html
+  shell:
+    executable: /bin/bash
+    cmd: |
+      set -e
+      cd {{ rally_results_dir }}
+      rally task report --html-static --out report.html
 
 - name: Show detailed info about task
   become: True

tests/unit/embedcharts/test_osprofilerchart.py

@@ -12,45 +12,193 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
+import datetime as dt
+import os
+
 import mock
-from rally_openstack.embedcharts.osprofilerchart import OSProfilerChart
+
+from rally_openstack.embedcharts import osprofilerchart as osp_chart
 from tests.unit import test
 
 
+PATH = "rally_openstack.embedcharts.osprofilerchart"
+CHART_PATH = "%s.OSProfilerChart" % PATH
+
+
 class OSProfilerChartTestCase(test.TestCase):
 
-    class OSProfilerChart(OSProfilerChart):
+    def test__datetime_json_serialize(self):
-        widget = "OSProfiler"
+        ts = dt.datetime(year=2018, month=7, day=3, hour=2)
+        self.assertEqual("2018-07-03T02:00:00",
+                         osp_chart._datetime_json_serialize(ts))
+        self.assertEqual("A", osp_chart._datetime_json_serialize("A"))
 
-    @mock.patch("osprofiler.drivers.base.get_driver")
+    def test__return_raw_response_for_complete_data(self):
-    def test_get_osprofiler_data(self, mock_get_driver):
+        title = "TITLE"
-        engine = mock.Mock()
+        trace_id = "trace-id"
-        attrs = {"get_report.return_value": "html"}
+        r = osp_chart.OSProfilerChart._return_raw_response_for_complete_data(
-        engine.configure_mock(**attrs)
+            {"title": title, "data": {"trace_id": trace_id}}
-        mock_get_driver.return_value = engine
+        )
+        self.assertEqual(
+            {"title": title, "widget": "TextArea", "data": [trace_id]},
+            r
+        )
 
-        data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
+    def test__generate_osprofiler_report(self):
-        return_data = OSProfilerChart.render_complete_data(data)
+        data = {"ts": dt.datetime(year=2018, month=7, day=3, hour=2)}
-        self.assertEqual("EmbedChart", return_data["widget"])
-        self.assertEqual("a : 1", return_data["title"])
 
-        data = {"data": {"conn_str": None, "trace_id": ["1"]}, "title": "a"}
+        mock_open = mock.mock_open(read_data="local=$LOCAL | data=$DATA")
-        return_data = OSProfilerChart.render_complete_data(data)
+        with mock.patch.object(osp_chart, "open", mock_open):
-        self.assertEqual("TextArea", return_data["widget"])
+            r = osp_chart.OSProfilerChart._generate_osprofiler_report(data)
-        self.assertEqual(["1"], return_data["data"])
+        self.assertEqual(
-        self.assertEqual("a", return_data["title"])
+            "local=false | data={\n    \"ts\": \"2018-07-03T02:00:00\"\n}",
+            r
+        )
+        self.assertEqual(1, mock_open.call_count)
+        m_args, _m_kwargs = mock_open.call_args_list[0]
+        self.assertTrue(os.path.exists(m_args[0]))
 
-        mock_get_driver.side_effect = Exception
+    def test__fetch_osprofiler_data(self):
-        data = {"data": {"conn_str": "a", "trace_id": ["1"]}, "title": "a"}
+        connection_str = "https://example.com"
-        return_data = OSProfilerChart.render_complete_data(data)
+        trace_id = "trace-id"
-        self.assertEqual("TextArea", return_data["widget"])
-        self.assertEqual(["1"], return_data["data"])
-        self.assertEqual("a", return_data["title"])
 
-    def test_datetime_json_serialize(self):
+        mock_osp_drivers = mock.Mock()
-        from rally_openstack.embedcharts.osprofilerchart \
+        mock_osp_driver = mock_osp_drivers.base
-            import _datetime_json_serialize
+        with mock.patch.dict(
-        A = mock.Mock()
+                "sys.modules", {"osprofiler.drivers": mock_osp_drivers}):
-        B = A.isoformat()
+            r = osp_chart.OSProfilerChart._fetch_osprofiler_data(
-        self.assertEqual(B, _datetime_json_serialize(A))
+                connection_str, trace_id)
-        self.assertEqual("C", _datetime_json_serialize("C"))
+            self.assertIsNotNone(r)
+
+        mock_osp_driver.get_driver.assert_called_once_with(connection_str)
+        engine = mock_osp_driver.get_driver.return_value
+        engine.get_report.assert_called_once_with(trace_id)
+        self.assertEqual(engine.get_report.return_value, r)
+
+        mock_osp_driver.get_driver.side_effect = Exception("Something")
+        with mock.patch.dict(
+                "sys.modules", {"osprofiler.drivers": mock_osp_drivers}):
+            r = osp_chart.OSProfilerChart._fetch_osprofiler_data(
+                connection_str, trace_id)
+            self.assertIsNone(r)
+
+    @mock.patch("%s.OutputEmbeddedExternalChart" % PATH)
+    @mock.patch("%s.OutputEmbeddedChart" % PATH)
+    @mock.patch("%s._return_raw_response_for_complete_data" % CHART_PATH)
+    @mock.patch("%s._fetch_osprofiler_data" % CHART_PATH)
+    @mock.patch("%s._generate_osprofiler_report" % CHART_PATH)
+    def test_render_complete_data(
+            self, mock__generate_osprofiler_report,
+            mock__fetch_osprofiler_data,
+            mock__return_raw_response_for_complete_data,
+            mock_output_embedded_chart,
+            mock_output_embedded_external_chart
+    ):
+        trace_id = "trace-id"
+        title = "TITLE"
+
+        # case 1: no connection-id, so data fpr text chart should be returned
+        pdata = {"data": {"trace_id": trace_id}, "title": title}
+        self.assertEqual(
+            mock__return_raw_response_for_complete_data.return_value,
+            osp_chart.OSProfilerChart.render_complete_data(
+                copy.deepcopy(pdata))
+        )
+        mock__return_raw_response_for_complete_data.assert_called_once_with(
+            pdata
+        )
+        mock__return_raw_response_for_complete_data.reset_mock()
+
+        # case 2: check support for an old format when `trace_id` key is a list
+        pdata = {"data": {"trace_id": [trace_id]}, "title": title}
+        self.assertEqual(
+            mock__return_raw_response_for_complete_data.return_value,
+            osp_chart.OSProfilerChart.render_complete_data(
+                copy.deepcopy(pdata))
+        )
+        pdata["data"]["trace_id"] = pdata["data"]["trace_id"][0]
+        mock__return_raw_response_for_complete_data.assert_called_once_with(
+            pdata
+        )
+        mock__return_raw_response_for_complete_data.reset_mock()
+
+        # case 3: connection-id is provided, but osp backed is not available
+        mock__fetch_osprofiler_data.return_value = None
+        pdata = {"data": {"trace_id": trace_id, "conn_str": "conn"},
+                 "title": title}
+        self.assertEqual(
+            mock__return_raw_response_for_complete_data.return_value,
+            osp_chart.OSProfilerChart.render_complete_data(
+                copy.deepcopy(pdata))
+        )
+        mock__return_raw_response_for_complete_data.assert_called_once_with(
+            pdata
+        )
+        mock__return_raw_response_for_complete_data.reset_mock()
+
+        # case 4: connection-id is provided
+        mock__fetch_osprofiler_data.return_value = "OSP_DATA"
+        mock__generate_osprofiler_report.return_value = "DD"
+        pdata = {"data": {"trace_id": trace_id, "conn_str": "conn"},
+                 "title": title}
+        self.assertEqual(
+            mock_output_embedded_chart.render_complete_data.return_value,
+            osp_chart.OSProfilerChart.render_complete_data(
+                copy.deepcopy(pdata))
+        )
+        mock_output_embedded_chart.render_complete_data.\
+            assert_called_once_with({"title": "%s : %s" % (title, trace_id),
+                                     "widget": "EmbeddedChart",
+                                     "data": "DD"})
+        self.assertFalse(mock__return_raw_response_for_complete_data.called)
+        mock_output_embedded_chart.render_complete_data.reset_mock()
+
+        # case 5: connection-id is provided with workload-id an
+        pdata = {"data": {"trace_id": trace_id,
+                          "conn_str": "conn",
+                          "workload_uuid": "W_ID",
+                          "iteration": 777},
+                 "title": title}
+
+        mock_open = mock.mock_open()
+        with mock.patch.object(osp_chart, "open", mock_open):
+            with mock.patch("%s.CONF.openstack" % PATH) as mock_cfg_os:
+                mock_cfg_os.osprofiler_chart_mode = "/path"
+
+                r = osp_chart.OSProfilerChart.render_complete_data(
+                    copy.deepcopy(pdata))
+
+        mock_external_chat = mock_output_embedded_external_chart
+        self.assertEqual(
+            mock_external_chat.render_complete_data.return_value,
+            r
+        )
+        mock_external_chat.render_complete_data.\
+            assert_called_once_with({"title": "%s : %s" % (title, trace_id),
+                                     "widget": "EmbeddedChart",
+                                     "data": "/path/w_W_ID-777.html"})
+        self.assertFalse(mock__return_raw_response_for_complete_data.called)
+
+        # case 6: rally < 1.5.0
+        pdata = {"data": {"trace_id": trace_id,
+                          "conn_str": "conn",
+                          "workload_uuid": "W_ID",
+                          "iteration": 777},
+                 "title": title}
+
+        mock_rally_os = mock.Mock()
+        mock_rally_os.__rally_version__ = (1, 4, 0)
+
+        with mock.patch.object(osp_chart, "rally_openstack") as m:
+            m.__rally_version__ = (1, 4, 0)
+            with mock.patch("%s.CONF.openstack" % PATH) as mock_cfg_os:
+                mock_cfg_os.osprofiler_chart_mode = "/path"
+
+                r = osp_chart.OSProfilerChart.render_complete_data(
+                    copy.deepcopy(pdata))
+                self.assertEqual({
+                    "title": "%s : %s" % (title, trace_id),
+                    "widget": "EmbeddedChart",
+                    "data": "DD"
+                }, r)

tests/unit/test_scenario.py

@@ -110,12 +110,17 @@ class OpenStackScenarioTestCase(test.TestCase):
                            mock_profiler_get,
                            mock_profiler_init):
         for user, credential in users_credentials:
-            self.context.update({user: {"credential": credential}})
+            self.context.update({user: {"credential": credential},
+                                 "iteration": 0})
         base_scenario.OpenStackScenario(self.context)
-        self.assertEqual(expected_call_count,
+
-                         mock_profiler_init.call_count)
+        if expected_call_count:
-        self.assertEqual([mock.call()] * expected_call_count,
+            mock_profiler_init.assert_called_once_with(
-                         mock_profiler_get.call_args_list)
+                CREDENTIAL_WITH_HMAC["profiler_hmac_key"])
+            mock_profiler_get.assert_called_once_with()
+        else:
+            self.assertFalse(mock_profiler_init.called)
+            self.assertFalse(mock_profiler_get.called)
 
     def test__choose_user_random(self):
         users = [{"credential": mock.Mock(), "tenant_id": "foo"}