Browse Source

Init profiler in for a new thread in post_tx_queue.py

* Initialization of profiler was also missing for a thread
  spawned within post_tx_queue.py so we were loosing important
  profiling info
* Changed the profiler test since its logic was already obsolete.
  Now we initialize profiler in every thread so the only reason to
  not get any profiler traces when a workflow completed is
  "enabled = False" in the "profiler" group in the configuration.
* Added more profiler traces
* Small readability changes in the workflow language spec

Change-Id: I35e6711f8e10bb08d7e842f4bca8753b929328fd
tags/10.0.0.0b2
Renat Akhmerov 3 months ago
parent
commit
829e822581
5 changed files with 23 additions and 18 deletions
  1. +6
    -0
      mistral/engine/post_tx_queue.py
  2. +3
    -3
      mistral/lang/v2/workflows.py
  3. +4
    -14
      mistral/tests/unit/engine/test_profiler.py
  4. +5
    -0
      mistral/workflow/data_flow.py
  5. +5
    -1
      mistral/workflow/direct_workflow.py

+ 6
- 0
mistral/engine/post_tx_queue.py View File

@@ -15,7 +15,9 @@

import eventlet
import functools
from oslo_config import cfg
from oslo_log import log as logging
from osprofiler import profiler

from mistral import context
from mistral.db import utils as db_utils
@@ -88,6 +90,10 @@ def run(func):
auth_ctx = context.ctx() if context.has_ctx() else None

def _within_new_thread():
# This is a new thread so we need to init a profiler again.
if cfg.CONF.profiler.enabled:
profiler.init(cfg.CONF.profiler.hmac_keys)

old_auth_ctx = context.ctx() if context.has_ctx() else None

context.set_ctx(auth_ctx)


+ 3
- 3
mistral/lang/v2/workflows.py View File

@@ -298,7 +298,7 @@ class DirectWorkflowSpec(WorkflowSpec):
def get_on_error_clause(self, t_name):
result = []

on_clause = self.get_tasks()[t_name].get_on_error()
on_clause = self.get_task(t_name).get_on_error()

if on_clause:
result = on_clause.get_next()
@@ -317,7 +317,7 @@ class DirectWorkflowSpec(WorkflowSpec):
def get_on_success_clause(self, t_name):
result = []

on_clause = self.get_tasks()[t_name].get_on_success()
on_clause = self.get_task(t_name).get_on_success()

if on_clause:
result = on_clause.get_next()
@@ -336,7 +336,7 @@ class DirectWorkflowSpec(WorkflowSpec):
def get_on_complete_clause(self, t_name):
result = []

on_clause = self.get_tasks()[t_name].get_on_complete()
on_clause = self.get_task(t_name).get_on_complete()

if on_clause:
result = on_clause.get_next()


+ 4
- 14
mistral/tests/unit/engine/test_profiler.py View File

@@ -15,7 +15,6 @@
import mock

from oslo_config import cfg
from oslo_utils import uuidutils
import osprofiler

from mistral import context
@@ -36,8 +35,9 @@ class EngineProfilerTest(base.EngineTestCase):
def setUp(self):
super(EngineProfilerTest, self).setUp()

# Configure the osprofiler.
# Configure the profiler.
self.mock_profiler_log_func = mock.Mock(return_value=None)

osprofiler.notifier.set(self.mock_profiler_log_func)

self.ctx_serializer = context.RpcContextSerializer()
@@ -54,16 +54,6 @@ class EngineProfilerTest(base.EngineTestCase):

wf_service.create_workflows(wf_def)

ctx = {
'trace_info': {
'hmac_key': cfg.CONF.profiler.hmac_keys,
'base_id': uuidutils.generate_uuid(),
'parent_id': uuidutils.generate_uuid()
}
}

self.ctx_serializer.deserialize_context(ctx)

wf_ex = self.engine_client.start_workflow('wf')

self.assertIsNotNone(wf_ex)
@@ -74,6 +64,8 @@ class EngineProfilerTest(base.EngineTestCase):
self.assertGreater(self.mock_profiler_log_func.call_count, 0)

def test_no_profile_trace(self):
self.override_config('enabled', False, 'profiler')

wf_def = """
version: '2.0'
wf:
@@ -85,8 +77,6 @@ class EngineProfilerTest(base.EngineTestCase):

wf_service.create_workflows(wf_def)

self.ctx_serializer.deserialize_context({})

wf_ex = self.engine_client.start_workflow('wf')

self.assertIsNotNone(wf_ex)


+ 5
- 0
mistral/workflow/data_flow.py View File

@@ -15,6 +15,7 @@

from oslo_config import cfg
from oslo_log import log as logging
from osprofiler import profiler

from mistral import context as auth_ctx
from mistral.db.v2.sqlalchemy import models
@@ -228,6 +229,10 @@ def publish_variables(task_ex, task_spec):
# {'branch': {vars}, 'global': {vars}, 'atomic': {vars}}


@profiler.trace(
'data-flow-evaluate-task-outbound-context',
hide_args=True
)
def evaluate_task_outbound_context(task_ex):
"""Evaluates task outbound Data Flow context.



+ 5
- 1
mistral/workflow/direct_workflow.py View File

@@ -129,7 +129,7 @@ class DirectWorkflowController(base.WorkflowController):
if not (t_s or t_n in commands.ENGINE_CMD_CLS):
raise exc.WorkflowException("Task '%s' not found." % t_n)
elif not t_s:
t_s = self.wf_spec.get_tasks()[task_ex.name]
t_s = self.wf_spec.get_task(task_ex.name)

triggered_by = [
{
@@ -275,6 +275,7 @@ class DirectWorkflowController(base.WorkflowController):

return res and not task_ex.has_next_tasks

@profiler.trace('direct-wf-controller-find-next-tasks', hide_args=True)
def _find_next_tasks(self, task_ex, ctx):
t_n = task_ex.name
t_s = task_ex.state
@@ -294,18 +295,21 @@ class DirectWorkflowController(base.WorkflowController):
for name, cond, params in self.wf_spec.get_on_error_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)

result.append((name, params, 'on-error'))

if t_s == states.SUCCESS:
for name, cond, params in self.wf_spec.get_on_success_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)

result.append((name, params, 'on-success'))

if states.is_completed(t_s) and not states.is_cancelled(t_s):
for name, cond, params in self.wf_spec.get_on_complete_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)

result.append((name, params, 'on-complete'))

return result


Loading…
Cancel
Save