From 75517eb0e01c876894ddfd60d6a8f3441de4c38a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 24 Jul 2015 18:10:55 -0700 Subject: [PATCH] Use the node built-in 'dfs_iter' instead of recursion We can just use the non-recursive depth first iteration of nodes when scanning for atoms to select for a given scope level instead of using recursive calls to achieve the same effect. This makes it possible to have large and heavily nested flows that are not restricted by the python stack limit. Change-Id: I0d18565680f777adbdfca9d4983636c6b3e848da --- taskflow/engines/action_engine/scopes.py | 18 ++++----- taskflow/tests/unit/test_types.py | 24 +++++++++--- taskflow/types/tree.py | 48 +++++++++++++++++------- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/taskflow/engines/action_engine/scopes.py b/taskflow/engines/action_engine/scopes.py index 1d309d89..4dbfa76f 100644 --- a/taskflow/engines/action_engine/scopes.py +++ b/taskflow/engines/action_engine/scopes.py @@ -21,7 +21,11 @@ LOG = logging.getLogger(__name__) def _depth_first_reverse_iterate(node, idx=-1): - """Iterates connected (in reverse) nodes in tree (from starting node).""" + """Iterates connected (in reverse) nodes (from starting node). + + Jumps through nodes with ``FLOW`` ``kind`` attribute (does not yield + them back). + """ # Always go left to right, since right to left is the pattern order # and we want to go backwards and not forwards through that ordering... if idx == -1: @@ -29,15 +33,11 @@ def _depth_first_reverse_iterate(node, idx=-1): else: children_iter = reversed(node[0:idx]) for child in children_iter: - child_kind = child.metadata['kind'] - if child_kind == co.FLOW: + if child.metadata['kind'] == co.FLOW: # Jump through these... - # - # TODO(harlowja): make this non-recursive and remove this - # style of doing this when - # https://review.openstack.org/#/c/205731/ merges... - for atom in _depth_first_reverse_iterate(child): - yield atom + for child_child in child.dfs_iter(right_to_left=False): + if child_child.metadata['kind'] in co.ATOMS: + yield child_child.item else: yield child.item diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 79044923..9399c893 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -467,24 +467,38 @@ CEO self.assertEqual(set(['animal', 'reptile', 'mammal', 'horse', 'primate', 'monkey', 'human']), set(things)) - def test_dfs_itr_order(self): + def test_dfs_itr_left_to_right(self): + root = self._make_species() + it = root.dfs_iter(include_self=False, right_to_left=False) + things = list([n.item for n in it]) + self.assertEqual(['reptile', 'mammal', 'primate', + 'human', 'monkey', 'horse'], things) + + def test_dfs_itr_no_self(self): root = self._make_species() - things = list([n.item for n in root.dfs_iter(include_self=True)]) - self.assertEqual(['animal', 'mammal', 'horse', 'primate', - 'monkey', 'human', 'reptile'], things) things = list([n.item for n in root.dfs_iter(include_self=False)]) self.assertEqual(['mammal', 'horse', 'primate', 'monkey', 'human', 'reptile'], things) - def test_bfs_iter(self): + def test_bfs_itr(self): root = self._make_species() things = list([n.item for n in root.bfs_iter(include_self=True)]) self.assertEqual(['animal', 'reptile', 'mammal', 'primate', 'horse', 'human', 'monkey'], things) + + def test_bfs_itr_no_self(self): + root = self._make_species() things = list([n.item for n in root.bfs_iter(include_self=False)]) self.assertEqual(['reptile', 'mammal', 'primate', 'horse', 'human', 'monkey'], things) + def test_bfs_itr_right_to_left(self): + root = self._make_species() + it = root.bfs_iter(include_self=False, right_to_left=True) + things = list([n.item for n in it]) + self.assertEqual(['mammal', 'reptile', 'horse', + 'primate', 'monkey', 'human'], things) + class OrderedSetTest(test.TestCase): diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 94c009e1..56c96bbb 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -36,8 +36,9 @@ class FrozenNode(Exception): class _DFSIter(object): """Depth first iterator (non-recursive) over the child nodes.""" - def __init__(self, root, include_self=False): + def __init__(self, root, include_self=False, right_to_left=True): self.root = root + self.right_to_left = bool(right_to_left) self.include_self = bool(include_self) def __iter__(self): @@ -45,20 +46,28 @@ class _DFSIter(object): if self.include_self: stack.append(self.root) else: - stack.extend(self.root.reverse_iter()) + if self.right_to_left: + stack.extend(self.root.reverse_iter()) + else: + # Traverse the left nodes first to the right nodes. + stack.extend(iter(self.root)) while stack: - node = stack.pop() # Visit the node. + node = stack.pop() yield node - # Traverse the left & right subtree. - stack.extend(node.reverse_iter()) + if self.right_to_left: + stack.extend(node.reverse_iter()) + else: + # Traverse the left nodes first to the right nodes. + stack.extend(iter(node)) class _BFSIter(object): """Breadth first iterator (non-recursive) over the child nodes.""" - def __init__(self, root, include_self=False): + def __init__(self, root, include_self=False, right_to_left=False): self.root = root + self.right_to_left = bool(right_to_left) self.include_self = bool(include_self) def __iter__(self): @@ -66,13 +75,20 @@ class _BFSIter(object): if self.include_self: q.append(self.root) else: - q.extend(self.root.reverse_iter()) + if self.right_to_left: + q.extend(iter(self.root)) + else: + # Traverse the left nodes first to the right nodes. + q.extend(self.root.reverse_iter()) while q: - node = q.popleft() # Visit the node. + node = q.popleft() yield node - # Traverse the left & right subtree. - q.extend(node.reverse_iter()) + if self.right_to_left: + q.extend(iter(node)) + else: + # Traverse the left nodes first to the right nodes. + q.extend(node.reverse_iter()) class Node(object): @@ -361,10 +377,14 @@ class Node(object): raise ValueError("%s is not contained in any child" % (item)) return index_at - def dfs_iter(self, include_self=False): + def dfs_iter(self, include_self=False, right_to_left=True): """Depth first iteration (non-recursive) over the child nodes.""" - return _DFSIter(self, include_self=include_self) + return _DFSIter(self, + include_self=include_self, + right_to_left=right_to_left) - def bfs_iter(self, include_self=False): + def bfs_iter(self, include_self=False, right_to_left=False): """Breadth first iteration (non-recursive) over the child nodes.""" - return _BFSIter(self, include_self=include_self) + return _BFSIter(self, + include_self=include_self, + right_to_left=right_to_left)