diff --git a/doc/source/img/engine_states.png b/doc/source/img/engine_states.png
deleted file mode 100644
index 3fc83da3..00000000
Binary files a/doc/source/img/engine_states.png and /dev/null differ
diff --git a/doc/source/img/engine_states.svg b/doc/source/img/engine_states.svg
new file mode 100644
index 00000000..497c31ef
--- /dev/null
+++ b/doc/source/img/engine_states.svg
@@ -0,0 +1,8 @@
+
+
+
+
+
diff --git a/doc/source/img/engine_states.txt b/doc/source/img/engine_states.txt
deleted file mode 100644
index 3b33255e..00000000
--- a/doc/source/img/engine_states.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# Created using web sequence diagrams.
-#
-# https://www.websequencediagrams.com/
-
-note over RESUMING
-Running starts here
-end note
-
-RESUMING->SCHEDULING: Resumes and \nschedules initial tasks.
-SCHEDULING->WAITING: Waits for any \nfuture to complete.
-WAITING->WAITING: Continue waiting for \nfuture to complete.
-WAITING->ANALYZING: Analyze future results.
-ANALYZING->SCHEDULING: Schedules next set of tasks.
diff --git a/doc/source/img/flow_states.png b/doc/source/img/flow_states.png
deleted file mode 100644
index 8ee0cb2a..00000000
Binary files a/doc/source/img/flow_states.png and /dev/null differ
diff --git a/doc/source/img/flow_states.svg b/doc/source/img/flow_states.svg
new file mode 100644
index 00000000..c6d9825e
--- /dev/null
+++ b/doc/source/img/flow_states.svg
@@ -0,0 +1,8 @@
+
+
+
+
+
diff --git a/doc/source/img/retry_states.png b/doc/source/img/retry_states.png
deleted file mode 100644
index ccab5c6b..00000000
Binary files a/doc/source/img/retry_states.png and /dev/null differ
diff --git a/doc/source/img/retry_states.svg b/doc/source/img/retry_states.svg
new file mode 100644
index 00000000..014516e0
--- /dev/null
+++ b/doc/source/img/retry_states.svg
@@ -0,0 +1,8 @@
+
+
+
+
+
diff --git a/doc/source/img/task_states.png b/doc/source/img/task_states.png
deleted file mode 100644
index 6654ded3..00000000
Binary files a/doc/source/img/task_states.png and /dev/null differ
diff --git a/doc/source/img/task_states.svg b/doc/source/img/task_states.svg
new file mode 100644
index 00000000..f40501ac
--- /dev/null
+++ b/doc/source/img/task_states.svg
@@ -0,0 +1,8 @@
+
+
+
+
+
diff --git a/doc/source/states.rst b/doc/source/states.rst
index 34841f4d..02fcaf15 100644
--- a/doc/source/states.rst
+++ b/doc/source/states.rst
@@ -7,46 +7,40 @@ States
Engine
======
-.. image:: img/engine_states.png
- :height: 265px
- :align: right
+.. image:: img/engine_states.svg
+ :width: 660px
+ :align: left
:alt: Action engine state transitions
-Executing
----------
+**RESUMING** - Prepares flow & atoms to be resumed.
-**RESUMING** - Prepare flow to be resumed.
+**SCHEDULING** - Schedules and submits atoms to be worked on.
-**SCHEDULING** - Schedule nodes to be worked on.
+**WAITING** - Wait for atoms to finish executing.
-**WAITING** - Wait for nodes to finish executing.
+**ANALYZING** - Analyzes and processes result/s of atom completion.
-**ANALYZING** - Analyze and process result/s of node completion.
+**SUCCESS** - Completed successfully.
-End
----
-
-**SUCCESS** - Engine completed successfully.
-
-**REVERTED** - Engine reverting was induced and all nodes were not completed
+**REVERTED** - Reverting was induced and all atoms were **not** completed
successfully.
-**SUSPENDED** - Engine was suspended while running..
+**SUSPENDED** - Suspended while running.
Flow
====
-.. image:: img/flow_states.png
- :height: 400px
- :align: right
+.. image:: img/flow_states.svg
+ :width: 660px
+ :align: left
:alt: Flow state transitions
**PENDING** - A flow starts its life in this state.
**RUNNING** - In this state flow makes a progress, executes and/or reverts its
-tasks.
+atoms.
-**SUCCESS** - Once all tasks have finished successfully the flow transitions to
+**SUCCESS** - Once all atoms have finished successfully the flow transitions to
the SUCCESS state.
**REVERTED** - The flow transitions to this state when it has been reverted
@@ -57,14 +51,14 @@ after the failure.
**SUSPENDING** - In the RUNNING state the flow can be suspended. When this
happens, flow transitions to the SUSPENDING state immediately. In that state
-the engine running the flow waits for running tasks to finish (since the engine
-can not preempt tasks that are active).
+the engine running the flow waits for running atoms to finish (since the engine
+can not preempt atoms that are active).
-**SUSPENDED** - When no tasks are running and all results received so far are
+**SUSPENDED** - When no atoms are running and all results received so far are
saved, the flow transitions from the SUSPENDING state to SUSPENDED. Also it may
-go to the SUCCESS state if all tasks were in fact ran, or to the REVERTED state
-if the flow was reverting and all tasks were reverted while the engine was
-waiting for running tasks to finish, or to the FAILURE state if tasks were run
+go to the SUCCESS state if all atoms were in fact ran, or to the REVERTED state
+if the flow was reverting and all atoms were reverted while the engine was
+waiting for running atoms to finish, or to the FAILURE state if atoms were run
or reverted and some of them failed.
**RESUMING** - When the flow is interrupted 'in a hard way' (e.g. server
@@ -79,24 +73,25 @@ From the SUCCESS, FAILURE or REVERTED states the flow can be ran again (and
thus it goes back into the RUNNING state). One of the possible use cases for
this transition is to allow for alteration of a flow or flow details associated
with a previously ran flow after the flow has finished, and client code wants
-to ensure that each task from this new (potentially updated) flow has its
+to ensure that each atom from this new (potentially updated) flow has its
chance to run.
.. note::
The current code also contains strong checks during each flow state
- transition using the model described above and raises the InvalidState
- exception if an invalid transition is attempted. This exception being
- triggered usually means there is some kind of bug in the engine code or some
- type of misuse/state violation is occurring, and should be reported as such.
+ transition using the model described above and raises the
+ :py:class:`~taskflow.exceptions.InvalidState` exception if an invalid
+ transition is attempted. This exception being triggered usually means there
+ is some kind of bug in the engine code or some type of misuse/state violation
+ is occurring, and should be reported as such.
Task
====
-.. image:: img/task_states.png
- :height: 265px
- :align: right
+.. image:: img/task_states.svg
+ :width: 660px
+ :align: left
:alt: Task state transitions
**PENDING** - When a task is added to a flow, it starts in the PENDING state,
@@ -105,7 +100,8 @@ on to complete. The task transitions to the PENDING state after it was
reverted and its flow was restarted or retried.
**RUNNING** - When flow starts to execute the task, it transitions to the
-RUNNING state, and stays in this state until its execute() method returns.
+RUNNING state, and stays in this state until its
+:py:meth:`execute() ` method returns.
**SUCCESS** - The task transitions to this state after it was finished
successfully.
@@ -115,20 +111,20 @@ error. When the flow containing this task is being reverted, all its tasks are
walked in particular order.
**REVERTING** - The task transitions to this state when the flow starts to
-revert it and its revert() method is called. Only tasks in the SUCCESS or
-FAILURE state can be reverted. If this method fails (raises exception), task
-goes to the FAILURE state.
+revert it and its :py:meth:`revert() ` method
+is called. Only tasks in the SUCCESS or FAILURE state can be reverted. If this
+method fails (raises exception), the task goes to the FAILURE state.
-**REVERTED** - The task that has been reverted appears it this state.
+**REVERTED** - A task that has been reverted appears in this state.
Retry
=====
-.. image:: img/retry_states.png
- :height: 275px
- :align: right
- :alt: Task state transitions
+.. image:: img/retry_states.svg
+ :width: 660px
+ :align: left
+ :alt: Retry state transitions
Retry has the same states as a task and one additional state.
@@ -138,7 +134,8 @@ on to complete. The retry transitions to the PENDING state after it was
reverted and its flow was restarted or retried.
**RUNNING** - When flow starts to execute the retry, it transitions to the
-RUNNING state, and stays in this state until its execute() method returns.
+RUNNING state, and stays in this state until its
+:py:meth:`execute() ` method returns.
**SUCCESS** - The retry transitions to this state after it was finished
successfully.
@@ -148,14 +145,12 @@ error. When the flow containing this retry is being reverted, all its tasks are
walked in particular order.
**REVERTING** - The retry transitions to this state when the flow starts to
-revert it and its revert() method is called. Only retries in SUCCESS or FAILURE
-state can be reverted. If this method fails (raises exception), task goes to
-the FAILURE.
+revert it and its :py:meth:`revert() ` method is
+called. Only retries in SUCCESS or FAILURE state can be reverted. If this
+method fails (raises exception), the retry goes to the FAILURE state.
-**REVERTED** - The retry that has been reverted appears it this state.
+**REVERTED** - A retry that has been reverted appears in this state.
**RETRYING** - If flow that is managed by the current retry was failed and
-reverted, the retry prepares it for the next run and transitions to the
+reverted, the engine prepares it for the next run and transitions to the
RETRYING state.
-
-
diff --git a/tools/generate_states.sh b/tools/generate_states.sh
new file mode 100755
index 00000000..2da75817
--- /dev/null
+++ b/tools/generate_states.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -u
+xsltproc=`which xsltproc`
+if [ -z "$xsltproc" ]; then
+ echo "Please install xsltproc before continuing."
+ exit 1
+fi
+
+set -e
+if [ ! -d "$PWD/.diagram-tools" ]; then
+ git clone "https://github.com/vidarh/diagram-tools.git" "$PWD/.diagram-tools"
+fi
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+img_dir="$script_dir/../doc/source/img"
+
+echo "---- Updating task state diagram ----"
+python $script_dir/state_graph.py -t -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/task_states.svg
+
+echo "---- Updating flow state diagram ----"
+python $script_dir/state_graph.py -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/flow_states.svg
+
+echo "---- Updating engine state diagram ----"
+python $script_dir/state_graph.py -e -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/engine_states.svg
+
+echo "---- Updating retry state diagram ----"
+python $script_dir/state_graph.py -r -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/retry_states.svg
diff --git a/tools/state_graph.py b/tools/state_graph.py
old mode 100644
new mode 100755
index f6a2057d..77b85636
--- a/tools/state_graph.py
+++ b/tools/state_graph.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+import optparse
import os
import sys
@@ -7,41 +8,13 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir))
sys.path.insert(0, top_dir)
-import optparse
-import subprocess
-import tempfile
+import networkx as nx
+
+# To get this installed you may have to follow:
+# https://code.google.com/p/pydot/issues/detail?id=93 (until fixed).
+import pydot
from taskflow import states
-from taskflow.types import graph as gr
-
-
-def mini_exec(cmd, ok_codes=(0,)):
- stdout = subprocess.PIPE
- stderr = subprocess.PIPE
- proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, stdin=None)
- (stdout, stderr) = proc.communicate()
- rc = proc.returncode
- if rc not in ok_codes:
- raise RuntimeError("Could not run %s [%s]\nStderr: %s"
- % (cmd, rc, stderr))
- return (stdout, stderr)
-
-
-def make_svg(graph, output_filename, output_format):
- # NOTE(harlowja): requires pydot!
- gdot = graph.export_to_dot()
- if output_format == 'dot':
- output = gdot
- elif output_format in ('svg', 'svgz', 'png'):
- with tempfile.NamedTemporaryFile(suffix=".dot") as fh:
- fh.write(gdot)
- fh.flush()
- cmd = ['dot', '-T%s' % output_format, fh.name]
- output, _stderr = mini_exec(cmd)
- else:
- raise ValueError('Unknown format: %s' % output_filename)
- with open(output_filename, "wb") as fh:
- fh.write(output)
def main():
@@ -52,6 +25,14 @@ def main():
action='store_true',
help="use task state transitions",
default=False)
+ parser.add_option("-r", "--retries", dest="retries",
+ action='store_true',
+ help="use retry state transitions",
+ default=False)
+ parser.add_option("-e", "--engines", dest="engines",
+ action='store_true',
+ help="use engine state transitions",
+ default=False)
parser.add_option("-T", "--format", dest="format",
help="output in given format",
default='svg')
@@ -60,20 +41,90 @@ def main():
if options.filename is None:
options.filename = 'states.%s' % options.format
- g = gr.DiGraph(name="State transitions")
- if not options.tasks:
- source = states._ALLOWED_FLOW_TRANSITIONS
+ types = [options.engines, options.retries, options.tasks]
+ if sum([int(i) for i in types]) > 1:
+ parser.error("Only one of task/retry/engines may be specified.")
+
+ disallowed = set()
+ start_node = states.PENDING
+ if options.tasks:
+ source = list(states._ALLOWED_TASK_TRANSITIONS)
+ source_type = "Tasks"
+ disallowed.add(states.RETRYING)
+ elif options.retries:
+ source = list(states._ALLOWED_TASK_TRANSITIONS)
+ source_type = "Retries"
+ elif options.engines:
+ # TODO(harlowja): place this in states.py
+ source = [
+ (states.RESUMING, states.SCHEDULING),
+ (states.SCHEDULING, states.WAITING),
+ (states.WAITING, states.ANALYZING),
+ (states.ANALYZING, states.SCHEDULING),
+ (states.ANALYZING, states.WAITING),
+ ]
+ for u in (states.SCHEDULING, states.ANALYZING):
+ for v in (states.SUSPENDED, states.SUCCESS, states.REVERTED):
+ source.append((u, v))
+ source_type = "Engines"
+ start_node = states.RESUMING
else:
- source = states._ALLOWED_TASK_TRANSITIONS
+ source = list(states._ALLOWED_FLOW_TRANSITIONS)
+ source_type = "Flow"
+
+ transitions = nx.DiGraph()
for (u, v) in source:
- if not g.has_node(u):
- g.add_node(u)
- if not g.has_node(v):
- g.add_node(v)
- g.add_edge(u, v)
- make_svg(g, options.filename, options.format)
+ if u not in disallowed:
+ transitions.add_node(u)
+ if v not in disallowed:
+ transitions.add_node(v)
+ for (u, v) in source:
+ if not transitions.has_node(u) or not transitions.has_node(v):
+ continue
+ transitions.add_edge(u, v)
+
+ graph_name = "%s states" % source_type
+ g = pydot.Dot(graph_name=graph_name, rankdir='LR',
+ nodesep='0.25', overlap='false',
+ ranksep="0.5", size="11x8.5",
+ splines='true', ordering='in')
+ node_attrs = {
+ 'fontsize': '11',
+ }
+ nodes = {}
+ nodes_order = []
+ edges_added = []
+ for (u, v) in nx.bfs_edges(transitions, source=start_node):
+ if u not in nodes:
+ nodes[u] = pydot.Node(u, **node_attrs)
+ g.add_node(nodes[u])
+ nodes_order.append(u)
+ if v not in nodes:
+ nodes[v] = pydot.Node(v, **node_attrs)
+ g.add_node(nodes[v])
+ nodes_order.append(v)
+ for u in nodes_order:
+ for v in transitions.successors_iter(u):
+ if (u, v) not in edges_added:
+ g.add_edge(pydot.Edge(nodes[u], nodes[v]))
+ edges_added.append((u, v))
+ start = pydot.Node("__start__", shape="point", width="0.1",
+ xlabel='start', fontcolor='green', **node_attrs)
+ g.add_node(start)
+ g.add_edge(pydot.Edge(start, nodes[start_node], style='dotted'))
+
+ print("*" * len(graph_name))
+ print(graph_name)
+ print("*" * len(graph_name))
+ print(g.to_string().strip())
+
+ g.write(options.filename, format=options.format)
print("Created %s at '%s'" % (options.format, options.filename))
+ # To make the svg more pretty use the following:
+ # $ xsltproc ../diagram-tools/notugly.xsl ./states.svg > pretty-states.svg
+ # Get diagram-tools from https://github.com/vidarh/diagram-tools.git
+
if __name__ == '__main__':
main()