diff --git a/taskflow/examples/dump_memory_backend.py b/taskflow/examples/dump_memory_backend.py new file mode 100644 index 00000000..7209dda0 --- /dev/null +++ b/taskflow/examples/dump_memory_backend.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import sys + +logging.basicConfig(level=logging.ERROR) + +self_dir = os.path.abspath(os.path.dirname(__file__)) +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) +sys.path.insert(0, self_dir) + +from taskflow import engines +from taskflow.patterns import linear_flow as lf +from taskflow.persistence import backends +from taskflow import task +from taskflow.utils import persistence_utils as pu + +# INTRO: in this example we create a dummy flow with a dummy task, and run +# it using a in-memory backend and pre/post run we dump out the contents +# of the in-memory backends tree structure (which can be quite useful to +# look at for debugging or other analysis). + + +class PrintTask(task.Task): + def execute(self): + print("Running '%s'" % self.name) + + +backend = backends.fetch({ + 'connection': 'memory://', +}) +book, flow_detail = pu.temporary_flow_detail(backend=backend) + +# Make a little flow and run it... +f = lf.Flow('root') +for alpha in ['a', 'b', 'c']: + f.add(PrintTask(alpha)) + +e = engines.load(f, flow_detail=flow_detail, + book=book, backend=backend) +e.compile() +e.prepare() + +print("----------") +print("Before run") +print("----------") +print(backend.memory.pformat()) +print("----------") + +e.run() + +print("---------") +print("After run") +print("---------") +entries = [os.path.join(backend.memory.root_path, child) + for child in backend.memory.ls(backend.memory.root_path)] +while entries: + path = entries.pop() + value = backend.memory[path] + if value: + print("%s -> %s" % (path, value)) + else: + print("%s" % (path)) + entries.extend(os.path.join(path, child) + for child in backend.memory.ls(path)) diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index 0f5e22b4..c1faae84 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -26,17 +26,20 @@ from taskflow.utils import lock_utils class Filesystem(object): - """An in-memory tree filesystem-like structure.""" + """An in-memory filesystem-like structure.""" - @staticmethod - def _normpath(path): - if not path.startswith(os.sep): + #: Root path of the in-memory filesystem. + root_path = os.sep + + @classmethod + def _normpath(cls, path): + if not path.startswith(cls.root_path): raise ValueError("This filesystem can only normalize absolute" " paths: '%s' is not valid" % path) return os.path.normpath(path) def __init__(self): - self._root = tree.Node(os.sep) + self._root = tree.Node(self.root_path, value=None) def ensure_path(self, path): path = self._normpath(path) @@ -49,7 +52,7 @@ class Filesystem(object): child_node = node.find(piece, only_direct=True, include_self=False) if child_node is None: - child_node = tree.Node(piece) + child_node = tree.Node(piece, value=None) node.add(child_node) node = child_node @@ -62,7 +65,7 @@ class Filesystem(object): node = node.find(piece, only_direct=True, include_self=False) if node is None: - raise exc.NotFound("Item not found %s" % path) + raise exc.NotFound("Path '%s' not found" % path) return node def _get_item(self, path, links=None): @@ -119,7 +122,7 @@ class Filesystem(object): only_direct=True, include_self=False) if child_node is None: - child_node = tree.Node(basename) + child_node = tree.Node(basename, value=None) parent_node.add(child_node) child_node.metadata['target'] = src_path