openstack/deb-python-taskflow
Code Issues Proposed changes

lazy loading for logbooks and flowdetails

Browse Source 
You can now specify a "lazy" argument when asking a
persistence backend for a logbook for flowdetail.
This will return a logbook with no flowdetails in it,
or a flowdetail with no atomdetails in it, respectively.

Change-Id: Ie81a19a2107b44e2c35c2a92507bec03eee55c50
This commit is contained in:
Dan Krause
2015-03-10 21:42:53 -05:00
parent f9e520679a
commit 9f64c47cb7
4 changed files with 65 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
taskflow/persistence/backends/impl_sqlalchemy.py
View File

@@ -527,7 +527,7 @@ class Connection(base.Connection):
raise exc.StorageFailure("Failed saving logbook"
" '%s'" % book.uuid, e)
def get_logbook(self, book_uuid):
def get_logbook(self, book_uuid, lazy=False):
try:
logbooks = self._tables.logbooks
with contextlib.closing(self._engine.connect()) as conn:
@@ -538,40 +538,42 @@ class Connection(base.Connection):
raise exc.NotFound("No logbook found with"
" uuid '%s'" % book_uuid)
book = self._converter.convert_book(row)
self._converter.populate_book(conn, book)
if not lazy:
self._converter.populate_book(conn, book)
return book
except sa_exc.DBAPIError as e:
raise exc.StorageFailure(
"Failed getting logbook '%s'" % book_uuid, e)
def get_logbooks(self):
def get_logbooks(self, lazy=False):
gathered = []
try:
with contextlib.closing(self._engine.connect()) as conn:
q = sql.select([self._tables.logbooks])
for row in conn.execute(q):
book = self._converter.convert_book(row)
self._converter.populate_book(conn, book)
if not lazy:
self._converter.populate_book(conn, book)
gathered.append(book)
except sa_exc.DBAPIError as e:
raise exc.StorageFailure("Failed getting logbooks", e)
for book in gathered:
yield book
def get_flows_for_book(self, book_uuid):
def get_flows_for_book(self, book_uuid, lazy=False):
gathered = []
try:
with contextlib.closing(self._engine.connect()) as conn:
for row in self._converter.flow_query_iter(conn, book_uuid):
flow_details = self._converter.populate_flow_detail(conn,
row)
gathered.append(flow_details)
for fd in self._converter.flow_query_iter(conn, book_uuid):
if not lazy:
self._converter.populate_flow_detail(conn, fd)
gathered.append(fd)
except sa_exc.DBAPIError as e:
raise exc.StorageFailure("Failed getting flow details", e)
for flow_details in gathered:
yield flow_details
def get_flow_details(self, fd_uuid):
def get_flow_details(self, fd_uuid, lazy=False):
try:
flowdetails = self._tables.flowdetails
with self._engine.begin() as conn:
@@ -581,7 +583,10 @@ class Connection(base.Connection):
if not row:
raise exc.NotFound("No flow details found with uuid"
" '%s'" % fd_uuid)
return self._converter.convert_flow_detail(row)
fd = self._converter.convert_flow_detail(row)
if not lazy:
self._converter.populate_flow_detail(conn, fd)
return fd
except sa_exc.SQLAlchemyError as e:
raise exc.StorageFailure("Failed getting flow details with"
" uuid '%s'" % fd_uuid, e)
@@ -601,5 +606,16 @@ class Connection(base.Connection):
raise exc.StorageFailure("Failed getting atom details with"
" uuid '%s'" % ad_uuid, e)
def get_atoms_for_flow(self, fd_uuid):
gathered = []
try:
with contextlib.closing(self._engine.connect()) as conn:
for ad in self._converter.atom_query_iter(conn, fd_uuid):
gathered.append(ad)
except sa_exc.DBAPIError as e:
raise exc.StorageFailure("Failed getting atom details", e)
for atom_details in gathered:
yield atom_details
def close(self):
pass

11
taskflow/persistence/base.py
View File

@@ -109,12 +109,12 @@ class Connection(object):
pass
@abc.abstractmethod
def get_logbook(self, book_uuid):
def get_logbook(self, book_uuid, lazy=False):
"""Fetches a logbook object matching the given uuid."""
pass
@abc.abstractmethod
def get_logbooks(self):
def get_logbooks(self, lazy=False):
"""Return an iterable of logbook objects."""
pass
@@ -124,7 +124,7 @@ class Connection(object):
pass
@abc.abstractmethod
def get_flow_details(self, fd_uuid):
def get_flow_details(self, fd_uuid, lazy=False):
"""Fetches a flowdetails object matching the given uuid."""
pass
@@ -133,6 +133,11 @@ class Connection(object):
"""Fetches a atomdetails object matching the given uuid."""
pass
@abc.abstractmethod
def get_atoms_for_flow(self, fd_uuid):
"""Return an iterable of atomdetails for a given flowdetails uuid."""
pass
def _format_atom(atom_detail):
return {

2
taskflow/persistence/path_based.py
View File

@@ -149,7 +149,7 @@ class PathBasedConnection(base.Connection):
def get_logbooks(self, lazy=False):
for book_uuid in self._get_children(self.book_path):
yield self.get_logbook(book_uuid, lazy)
yield self.get_logbook(book_uuid, lazy=lazy)
def get_logbook(self, book_uuid, lazy=False):
book_path = self._join_path(self.book_path, book_uuid)

29
taskflow/tests/unit/persistence/base.py
View File

@@ -130,6 +130,22 @@ class PersistenceTestMixin(object):
fd2 = lb2.find(fd.uuid)
self.assertEqual(fd2.meta.get('test'), 43)
def test_flow_detail_lazy_fetch(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = logbook.LogBook(name=lb_name, uuid=lb_id)
fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid())
td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
td.version = '4.2'
fd.add(td)
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
fd2 = conn.get_flow_details(fd.uuid, lazy=True)
self.assertEqual(0, len(fd2))
self.assertEqual(1, len(fd))
def test_task_detail_save(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
@@ -239,6 +255,19 @@ class PersistenceTestMixin(object):
self.assertEqual(1, len(lb))
self.assertEqual(fd.name, lb2.find(fd.uuid).name)
def test_logbook_lazy_fetch(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = logbook.LogBook(name=lb_name, uuid=lb_id)
fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id, lazy=True)
self.assertEqual(0, len(lb2))
self.assertEqual(1, len(lb))
def test_logbook_add_task_detail(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)