Merge "Add a in-memory backend dumping example"

taskflow/examples/dump_memory_backend.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import os
+import sys
+
+logging.basicConfig(level=logging.ERROR)
+
+self_dir = os.path.abspath(os.path.dirname(__file__))
+top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                       os.pardir,
+                                       os.pardir))
+sys.path.insert(0, top_dir)
+sys.path.insert(0, self_dir)
+
+from taskflow import engines
+from taskflow.patterns import linear_flow as lf
+from taskflow.persistence import backends
+from taskflow import task
+from taskflow.utils import persistence_utils as pu
+
+# INTRO: in this example we create a dummy flow with a dummy task, and run
+# it using a in-memory backend and pre/post run we dump out the contents
+# of the in-memory backends tree structure (which can be quite useful to
+# look at for debugging or other analysis).
+
+
+class PrintTask(task.Task):
+    def execute(self):
+        print("Running '%s'" % self.name)
+
+
+backend = backends.fetch({
+    'connection': 'memory://',
+})
+book, flow_detail = pu.temporary_flow_detail(backend=backend)
+
+# Make a little flow and run it...
+f = lf.Flow('root')
+for alpha in ['a', 'b', 'c']:
+    f.add(PrintTask(alpha))
+
+e = engines.load(f, flow_detail=flow_detail,
+                 book=book, backend=backend)
+e.compile()
+e.prepare()
+
+print("----------")
+print("Before run")
+print("----------")
+print(backend.memory.pformat())
+print("----------")
+
+e.run()
+
+print("---------")
+print("After run")
+print("---------")
+entries = [os.path.join(backend.memory.root_path, child)
+           for child in backend.memory.ls(backend.memory.root_path)]
+while entries:
+    path = entries.pop()
+    value = backend.memory[path]
+    if value:
+        print("%s -> %s" % (path, value))
+    else:
+        print("%s" % (path))
+    entries.extend(os.path.join(path, child)
+                   for child in backend.memory.ls(path))

taskflow/persistence/backends/impl_memory.py

@@ -26,17 +26,20 @@ from taskflow.utils import lock_utils
 
 
 class Filesystem(object):
-    """An in-memory tree filesystem-like structure."""
+    """An in-memory filesystem-like structure."""
 
-    @staticmethod
+    #: Root path of the in-memory filesystem.
-    def _normpath(path):
+    root_path = os.sep
-        if not path.startswith(os.sep):
+
+    @classmethod
+    def _normpath(cls, path):
+        if not path.startswith(cls.root_path):
             raise ValueError("This filesystem can only normalize absolute"
                              " paths: '%s' is not valid" % path)
         return os.path.normpath(path)
 
     def __init__(self):
-        self._root = tree.Node(os.sep)
+        self._root = tree.Node(self.root_path, value=None)
 
     def ensure_path(self, path):
         path = self._normpath(path)
@@ -49,7 +52,7 @@ class Filesystem(object):
             child_node = node.find(piece, only_direct=True,
                                    include_self=False)
             if child_node is None:
-                child_node = tree.Node(piece)
+                child_node = tree.Node(piece, value=None)
                 node.add(child_node)
             node = child_node
 
@@ -62,7 +65,7 @@ class Filesystem(object):
             node = node.find(piece, only_direct=True,
                              include_self=False)
             if node is None:
-                raise exc.NotFound("Item not found %s" % path)
+                raise exc.NotFound("Path '%s' not found" % path)
         return node
 
     def _get_item(self, path, links=None):
@@ -119,7 +122,7 @@ class Filesystem(object):
                                       only_direct=True,
                                       include_self=False)
         if child_node is None:
-            child_node = tree.Node(basename)
+            child_node = tree.Node(basename, value=None)
             parent_node.add(child_node)
         child_node.metadata['target'] = src_path