66a16b240a
Ensure users of this code/objects are aware that only posixpath style paths will work (and not the variant that is used in os.path which can change depending on operating system) as well as adding docstrings on other methods so that they show up in the generated docs. Change-Id: I5a21cb75b2452b9c9d65860b63658a9d0980025f
249 lines
8.3 KiB
Python
249 lines
8.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
# Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import contextlib
|
|
import copy
|
|
import posixpath as pp
|
|
|
|
from taskflow import exceptions as exc
|
|
from taskflow.persistence import path_based
|
|
from taskflow.types import tree
|
|
from taskflow.utils import lock_utils
|
|
|
|
|
|
class FakeFilesystem(object):
|
|
"""An in-memory filesystem-like structure.
|
|
|
|
This filesystem uses posix style paths **only** so users must be careful
|
|
to use the ``posixpath`` module instead of the ``os.path`` one which will
|
|
vary depending on the operating system which the active python is running
|
|
in (the decision to use ``posixpath`` was to avoid the path variations
|
|
which are not relevant in an implementation of a in-memory fake
|
|
filesystem).
|
|
|
|
Example usage:
|
|
|
|
>>> from taskflow.persistence.backends import impl_memory
|
|
>>> fs = impl_memory.FakeFilesystem()
|
|
>>> fs.ensure_path('/a/b/c')
|
|
>>> fs['/a/b/c'] = 'd'
|
|
>>> print(fs['/a/b/c'])
|
|
d
|
|
>>> del fs['/a/b/c']
|
|
>>> fs.ls("/a/b")
|
|
[]
|
|
"""
|
|
|
|
#: Root path of the in-memory filesystem.
|
|
root_path = pp.sep
|
|
|
|
@classmethod
|
|
def _normpath(cls, path):
|
|
if not path.startswith(cls.root_path):
|
|
raise ValueError("This filesystem can only normalize absolute"
|
|
" paths: '%s' is not valid" % path)
|
|
return pp.normpath(path)
|
|
|
|
def __init__(self, deep_copy=True):
|
|
self._root = tree.Node(self.root_path, value=None)
|
|
if deep_copy:
|
|
self._copier = copy.deepcopy
|
|
else:
|
|
self._copier = copy.copy
|
|
|
|
def ensure_path(self, path):
|
|
"""Ensure the path (and parents) exists."""
|
|
path = self._normpath(path)
|
|
# Ignore the root path as we already checked for that; and it
|
|
# will always exist/can't be removed anyway...
|
|
if path == self._root.item:
|
|
return
|
|
node = self._root
|
|
for piece in self._iter_pieces(path):
|
|
child_node = node.find(piece, only_direct=True,
|
|
include_self=False)
|
|
if child_node is None:
|
|
child_node = tree.Node(piece, value=None)
|
|
node.add(child_node)
|
|
node = child_node
|
|
|
|
def _fetch_node(self, path):
|
|
node = self._root
|
|
path = self._normpath(path)
|
|
if path == self._root.item:
|
|
return node
|
|
for piece in self._iter_pieces(path):
|
|
node = node.find(piece, only_direct=True,
|
|
include_self=False)
|
|
if node is None:
|
|
raise exc.NotFound("Path '%s' not found" % path)
|
|
return node
|
|
|
|
def _get_item(self, path, links=None):
|
|
node = self._fetch_node(path)
|
|
if 'target' in node.metadata:
|
|
# Follow the link (and watch out for loops)...
|
|
path = node.metadata['target']
|
|
if links is None:
|
|
links = []
|
|
if path in links:
|
|
raise ValueError("Recursive link following not"
|
|
" allowed (loop %s detected)"
|
|
% (links + [path]))
|
|
else:
|
|
links.append(path)
|
|
return self._get_item(path, links=links)
|
|
else:
|
|
return self._copier(node.metadata['value'])
|
|
|
|
def ls(self, path):
|
|
"""Return list of all children of the given path."""
|
|
return [node.item for node in self._fetch_node(path)]
|
|
|
|
def _iter_pieces(self, path, include_root=False):
|
|
if path == self._root.item:
|
|
# Check for this directly as the following doesn't work with
|
|
# split correctly:
|
|
#
|
|
# >>> path = "/"
|
|
# path.split(pp.sep)
|
|
# ['', '']
|
|
parts = []
|
|
else:
|
|
parts = path.split(pp.sep)[1:]
|
|
if include_root:
|
|
parts.insert(0, self._root.item)
|
|
for piece in parts:
|
|
yield piece
|
|
|
|
def __delitem__(self, path):
|
|
node = self._fetch_node(path)
|
|
if node is self._root:
|
|
raise ValueError("Can not delete '%s'" % self._root.item)
|
|
node.disassociate()
|
|
|
|
def pformat(self):
|
|
"""Pretty format this in-memory filesystem."""
|
|
return self._root.pformat()
|
|
|
|
def symlink(self, src_path, dest_path):
|
|
"""Link the destionation path to the source path."""
|
|
dest_path = self._normpath(dest_path)
|
|
src_path = self._normpath(src_path)
|
|
dirname, basename = pp.split(dest_path)
|
|
parent_node = self._fetch_node(dirname)
|
|
child_node = parent_node.find(basename,
|
|
only_direct=True,
|
|
include_self=False)
|
|
if child_node is None:
|
|
child_node = tree.Node(basename, value=None)
|
|
parent_node.add(child_node)
|
|
child_node.metadata['target'] = src_path
|
|
|
|
def __getitem__(self, path):
|
|
return self._get_item(path)
|
|
|
|
def __setitem__(self, path, value):
|
|
path = self._normpath(path)
|
|
value = self._copier(value)
|
|
try:
|
|
item_node = self._fetch_node(path)
|
|
item_node.metadata.update(value=value)
|
|
except exc.NotFound:
|
|
dirname, basename = pp.split(path)
|
|
parent_node = self._fetch_node(dirname)
|
|
parent_node.add(tree.Node(basename, value=value))
|
|
|
|
|
|
class MemoryBackend(path_based.PathBasedBackend):
|
|
"""A in-memory (non-persistent) backend.
|
|
|
|
This backend writes logbooks, flow details, and atom details to a
|
|
in-memory filesystem-like structure (rooted by the ``memory``
|
|
instance variable).
|
|
|
|
This backend does *not* provide true transactional semantics. It does
|
|
guarantee that there will be no inter-thread race conditions when
|
|
writing and reading by using a read/write locks.
|
|
"""
|
|
def __init__(self, conf=None):
|
|
super(MemoryBackend, self).__init__(conf)
|
|
if self._path is None:
|
|
self._path = pp.sep
|
|
self.memory = FakeFilesystem(deep_copy=self._conf.get('deep_copy',
|
|
True))
|
|
self.lock = lock_utils.ReaderWriterLock()
|
|
|
|
def get_connection(self):
|
|
return Connection(self)
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
|
|
class Connection(path_based.PathBasedConnection):
|
|
def __init__(self, backend):
|
|
super(Connection, self).__init__(backend)
|
|
self.upgrade()
|
|
|
|
@contextlib.contextmanager
|
|
def _memory_lock(self, write=False):
|
|
if write:
|
|
lock = self.backend.lock.write_lock
|
|
else:
|
|
lock = self.backend.lock.read_lock
|
|
with lock():
|
|
try:
|
|
yield
|
|
except exc.TaskFlowException as e:
|
|
raise
|
|
except Exception as e:
|
|
raise exc.StorageFailure("Storage backend internal error", e)
|
|
|
|
def _join_path(self, *parts):
|
|
return pp.join(*parts)
|
|
|
|
def _get_item(self, path):
|
|
with self._memory_lock():
|
|
return self.backend.memory[path]
|
|
|
|
def _set_item(self, path, value, transaction):
|
|
self.backend.memory[path] = value
|
|
|
|
def _del_tree(self, path, transaction):
|
|
del self.backend.memory[path]
|
|
|
|
def _get_children(self, path):
|
|
with self._memory_lock():
|
|
return self.backend.memory.ls(path)
|
|
|
|
def _ensure_path(self, path):
|
|
with self._memory_lock(write=True):
|
|
self.backend.memory.ensure_path(path)
|
|
|
|
def _create_link(self, src_path, dest_path, transaction):
|
|
self.backend.memory.symlink(src_path, dest_path)
|
|
|
|
@contextlib.contextmanager
|
|
def _transaction(self):
|
|
"""This just wraps a global write-lock."""
|
|
with self._memory_lock(write=True):
|
|
yield
|
|
|
|
def validate(self):
|
|
pass
|