From acd545f9c2974f378eb0af7da70512fc5ce19383 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 15 Aug 2013 16:54:00 -0700 Subject: [PATCH] File movements In order to rework some of the persistence layer I would like to move around some of the files first, keeping job specifics in a jobs folder. Having some of the items which are root level taskflow items (flow, task) be at the root of the hiearchy. Also for now until the celery work is commited move that, since it doesn't make sense in backends anyway. Change-Id: If6c149710b40f70d4ec69ee8e523defe8f5e766d --- taskflow/backends/celery/__init__.py | 17 -------- taskflow/backends/celery/celeryapp.py | 42 ------------------- taskflow/backends/celery/celeryconfig.py | 32 -------------- taskflow/{generics => }/flow.py | 0 taskflow/{generics => jobs}/__init__.py | 0 taskflow/{generics => jobs}/job.py | 0 taskflow/{generics => jobs}/jobboard.py | 0 taskflow/patterns/linear_flow.py | 2 +- .../sqlalchemy => persistence}/__init__.py | 1 - .../backends/__init__.py} | 1 - taskflow/{ => persistence}/backends/api.py | 5 ++- .../backends/memory}/__init__.py | 0 .../{ => persistence}/backends/memory/api.py | 8 ++-- .../backends/memory/memory.py | 14 +++---- .../backends/sqlalchemy}/__init__.py | 0 .../backends/sqlalchemy/api.py | 10 ++--- .../backends/sqlalchemy}/base.py | 0 .../backends/sqlalchemy/models.py | 2 +- .../backends/sqlalchemy/session.py | 2 +- .../{generics => persistence}/flowdetail.py | 2 +- taskflow/{generics => persistence}/logbook.py | 2 +- .../{generics => persistence}/taskdetail.py | 2 +- taskflow/{generics => }/task.py | 0 taskflow/tests/unit/memory_api/__init__.py | 2 +- .../unit/memory_api/test_flowdetail_api.py | 4 +- .../tests/unit/memory_api/test_logbook_api.py | 4 +- .../unit/memory_api/test_taskdetail_api.py | 4 +- taskflow/tests/unit/sql_db_api/__init__.py | 4 +- .../unit/sql_db_api/test_flowdetail_api.py | 4 +- .../tests/unit/sql_db_api/test_logbook_api.py | 4 +- .../unit/sql_db_api/test_taskdetail_api.py | 4 +- taskflow/tests/unit/test_linear_flow.py | 2 +- taskflow/tests/utils.py | 2 +- tools/pip-requires | 3 +- 34 files changed, 43 insertions(+), 136 deletions(-) delete mode 100644 taskflow/backends/celery/__init__.py delete mode 100644 taskflow/backends/celery/celeryapp.py delete mode 100644 taskflow/backends/celery/celeryconfig.py rename taskflow/{generics => }/flow.py (100%) rename taskflow/{generics => jobs}/__init__.py (100%) rename taskflow/{generics => jobs}/job.py (100%) rename taskflow/{generics => jobs}/jobboard.py (100%) rename taskflow/{backends/sqlalchemy => persistence}/__init__.py (92%) rename taskflow/{backends/migration.py => persistence/backends/__init__.py} (92%) rename taskflow/{ => persistence}/backends/api.py (98%) rename taskflow/{backends => persistence/backends/memory}/__init__.py (100%) rename taskflow/{ => persistence}/backends/memory/api.py (98%) rename taskflow/{ => persistence}/backends/memory/memory.py (93%) rename taskflow/{backends/memory => persistence/backends/sqlalchemy}/__init__.py (100%) rename taskflow/{ => persistence}/backends/sqlalchemy/api.py (98%) rename taskflow/{backends => persistence/backends/sqlalchemy}/base.py (100%) rename taskflow/{ => persistence}/backends/sqlalchemy/models.py (98%) rename taskflow/{ => persistence}/backends/sqlalchemy/session.py (98%) rename taskflow/{generics => persistence}/flowdetail.py (97%) rename taskflow/{generics => persistence}/logbook.py (97%) rename taskflow/{generics => persistence}/taskdetail.py (96%) rename taskflow/{generics => }/task.py (100%) diff --git a/taskflow/backends/celery/__init__.py b/taskflow/backends/celery/__init__.py deleted file mode 100644 index fbe1837c..00000000 --- a/taskflow/backends/celery/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/taskflow/backends/celery/celeryapp.py b/taskflow/backends/celery/celeryapp.py deleted file mode 100644 index e41d5d64..00000000 --- a/taskflow/backends/celery/celeryapp.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import traceback as tb - -from celery import signals - -LOG = logging.getLogger(__name__) - - -@signals.task_failure.connect -def task_error_handler(signal=None, sender=None, task_id=None, - exception=None, args=None, kwargs=None, - traceback=None, einfo=None): - """If a task errors out, log all error info""" - LOG.error('Task %s, id: %s, called with args: %s, and kwargs: %s' - 'failed with exception: %s' % (sender.name, task_id, - args, kwargs, exception)) - LOG.error('Trackeback: %s' % (tb.print_tb(traceback), )) - # TODO(jlucci): Auto-initiate rollback from failed task - - -@signals.task_success.connect -def task_success_handler(singal=None, sender=None, result=None): - """Save task results to WF.""" - pass diff --git a/taskflow/backends/celery/celeryconfig.py b/taskflow/backends/celery/celeryconfig.py deleted file mode 100644 index 5a64ecc0..00000000 --- a/taskflow/backends/celery/celeryconfig.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" Celery Configuration File """ - -import logging - -from taskflow.common import config - -from oslo.config import cfg - -config.register_celery_opts() - -LOG = logging.getLogger(__name__) -BROKER_URL = cfg.CFG('celery_mq') -CELERY_RESULT_BACKEND = "database" -CELERY_RESULT_DBURI = cfg.CFG('celery_backend') diff --git a/taskflow/generics/flow.py b/taskflow/flow.py similarity index 100% rename from taskflow/generics/flow.py rename to taskflow/flow.py diff --git a/taskflow/generics/__init__.py b/taskflow/jobs/__init__.py similarity index 100% rename from taskflow/generics/__init__.py rename to taskflow/jobs/__init__.py diff --git a/taskflow/generics/job.py b/taskflow/jobs/job.py similarity index 100% rename from taskflow/generics/job.py rename to taskflow/jobs/job.py diff --git a/taskflow/generics/jobboard.py b/taskflow/jobs/jobboard.py similarity index 100% rename from taskflow/generics/jobboard.py rename to taskflow/jobs/jobboard.py diff --git a/taskflow/patterns/linear_flow.py b/taskflow/patterns/linear_flow.py index 78aabc9a..a2da98c0 100644 --- a/taskflow/patterns/linear_flow.py +++ b/taskflow/patterns/linear_flow.py @@ -26,7 +26,7 @@ from taskflow import exceptions as exc from taskflow import states from taskflow import utils -from taskflow.generics import flow +from taskflow import flow LOG = logging.getLogger(__name__) diff --git a/taskflow/backends/sqlalchemy/__init__.py b/taskflow/persistence/__init__.py similarity index 92% rename from taskflow/backends/sqlalchemy/__init__.py rename to taskflow/persistence/__init__.py index 275e7e13..3a554a5f 100644 --- a/taskflow/backends/sqlalchemy/__init__.py +++ b/taskflow/persistence/__init__.py @@ -2,7 +2,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. # Copyright (C) 2013 Rackspace Hosting All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/taskflow/backends/migration.py b/taskflow/persistence/backends/__init__.py similarity index 92% rename from taskflow/backends/migration.py rename to taskflow/persistence/backends/__init__.py index 275e7e13..3a554a5f 100644 --- a/taskflow/backends/migration.py +++ b/taskflow/persistence/backends/__init__.py @@ -2,7 +2,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. # Copyright (C) 2013 Rackspace Hosting All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/taskflow/backends/api.py b/taskflow/persistence/backends/api.py similarity index 98% rename from taskflow/backends/api.py rename to taskflow/persistence/backends/api.py index a7b33dfe..f7879fc8 100644 --- a/taskflow/backends/api.py +++ b/taskflow/persistence/backends/api.py @@ -38,8 +38,9 @@ CONF.register_opts(db_opts) CONF.register_opts(mem_opts) IMPL = utils.LazyPluggable('mem_backend', - memory='taskflow.backends.memory.api', - sqlalchemy='taskflow.backends.sqlalchemy.api') + memory='taskflow.persistence.backends.memory.api', + sqlalchemy='taskflow.persistence.backends' + '.sqlalchemy.api') def configure(pivot='mem_backend'): diff --git a/taskflow/backends/__init__.py b/taskflow/persistence/backends/memory/__init__.py similarity index 100% rename from taskflow/backends/__init__.py rename to taskflow/persistence/backends/memory/__init__.py diff --git a/taskflow/backends/memory/api.py b/taskflow/persistence/backends/memory/api.py similarity index 98% rename from taskflow/backends/memory/api.py rename to taskflow/persistence/backends/memory/api.py index 5bf1976b..fdaa02fe 100644 --- a/taskflow/backends/memory/api.py +++ b/taskflow/persistence/backends/memory/api.py @@ -21,11 +21,11 @@ import logging -from taskflow.backends.memory import memory from taskflow import exceptions as exception -from taskflow.generics import flowdetail -from taskflow.generics import logbook -from taskflow.generics import taskdetail +from taskflow.persistence.backends.memory import memory +from taskflow.persistence import flowdetail +from taskflow.persistence import logbook +from taskflow.persistence import taskdetail from taskflow.utils import LockingDict LOG = logging.getLogger(__name__) diff --git a/taskflow/backends/memory/memory.py b/taskflow/persistence/backends/memory/memory.py similarity index 93% rename from taskflow/backends/memory/memory.py rename to taskflow/persistence/backends/memory/memory.py index c8ffcd71..805861d8 100644 --- a/taskflow/backends/memory/memory.py +++ b/taskflow/persistence/backends/memory/memory.py @@ -18,13 +18,13 @@ import contextlib -from taskflow.generics import flow -from taskflow.generics import flowdetail -from taskflow.generics import job -from taskflow.generics import jobboard -from taskflow.generics import logbook -from taskflow.generics import task -from taskflow.generics import taskdetail +from taskflow import flow +from taskflow.jobs import job +from taskflow.jobs import jobboard +from taskflow.persistence import flowdetail +from taskflow.persistence import logbook +from taskflow.persistence import taskdetail +from taskflow import task from taskflow import utils diff --git a/taskflow/backends/memory/__init__.py b/taskflow/persistence/backends/sqlalchemy/__init__.py similarity index 100% rename from taskflow/backends/memory/__init__.py rename to taskflow/persistence/backends/sqlalchemy/__init__.py diff --git a/taskflow/backends/sqlalchemy/api.py b/taskflow/persistence/backends/sqlalchemy/api.py similarity index 98% rename from taskflow/backends/sqlalchemy/api.py rename to taskflow/persistence/backends/sqlalchemy/api.py index 2aa0ee3d..6a378074 100644 --- a/taskflow/backends/sqlalchemy/api.py +++ b/taskflow/persistence/backends/sqlalchemy/api.py @@ -22,12 +22,12 @@ import logging from sqlalchemy import exc -from taskflow.backends.sqlalchemy import models -from taskflow.backends.sqlalchemy import session as sql_session from taskflow import exceptions as exception -from taskflow.generics import flowdetail -from taskflow.generics import logbook -from taskflow.generics import taskdetail +from taskflow.persistence.backends.sqlalchemy import models +from taskflow.persistence.backends.sqlalchemy import session as sql_session +from taskflow.persistence import flowdetail +from taskflow.persistence import logbook +from taskflow.persistence import taskdetail LOG = logging.getLogger(__name__) diff --git a/taskflow/backends/base.py b/taskflow/persistence/backends/sqlalchemy/base.py similarity index 100% rename from taskflow/backends/base.py rename to taskflow/persistence/backends/sqlalchemy/base.py diff --git a/taskflow/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py similarity index 98% rename from taskflow/backends/sqlalchemy/models.py rename to taskflow/persistence/backends/sqlalchemy/models.py index 3ac98539..f307978b 100644 --- a/taskflow/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -29,10 +29,10 @@ from sqlalchemy.orm import object_mapper, relationship from sqlalchemy import DateTime, ForeignKey from sqlalchemy import types as types -from taskflow.backends.sqlalchemy import session as sql_session from taskflow import exceptions as exception from taskflow.openstack.common import timeutils from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends.sqlalchemy import session as sql_session CONF = cfg.CONF BASE = declarative_base() diff --git a/taskflow/backends/sqlalchemy/session.py b/taskflow/persistence/backends/sqlalchemy/session.py similarity index 98% rename from taskflow/backends/sqlalchemy/session.py rename to taskflow/persistence/backends/sqlalchemy/session.py index be7320d0..39120a5a 100644 --- a/taskflow/backends/sqlalchemy/session.py +++ b/taskflow/persistence/backends/sqlalchemy/session.py @@ -29,7 +29,7 @@ from sqlalchemy.pool import NullPool from sqlalchemy import exc -from taskflow.backends import api as b_api +from taskflow.persistence.backends import api as b_api LOG = logging.getLogger(__name__) diff --git a/taskflow/generics/flowdetail.py b/taskflow/persistence/flowdetail.py similarity index 97% rename from taskflow/generics/flowdetail.py rename to taskflow/persistence/flowdetail.py index a8e5f626..ebcc2051 100644 --- a/taskflow/generics/flowdetail.py +++ b/taskflow/persistence/flowdetail.py @@ -18,8 +18,8 @@ from datetime import datetime -from taskflow.backends import api as b_api from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends import api as b_api class FlowDetail(object): diff --git a/taskflow/generics/logbook.py b/taskflow/persistence/logbook.py similarity index 97% rename from taskflow/generics/logbook.py rename to taskflow/persistence/logbook.py index d3ad2db5..82a38860 100644 --- a/taskflow/generics/logbook.py +++ b/taskflow/persistence/logbook.py @@ -18,8 +18,8 @@ from datetime import datetime -from taskflow.backends import api as b_api from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends import api as b_api class LogBook(object): diff --git a/taskflow/generics/taskdetail.py b/taskflow/persistence/taskdetail.py similarity index 96% rename from taskflow/generics/taskdetail.py rename to taskflow/persistence/taskdetail.py index 965c2d7c..f182eff6 100644 --- a/taskflow/generics/taskdetail.py +++ b/taskflow/persistence/taskdetail.py @@ -18,8 +18,8 @@ from datetime import datetime -from taskflow.backends import api as b_api from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends import api as b_api class TaskDetail(object): diff --git a/taskflow/generics/task.py b/taskflow/task.py similarity index 100% rename from taskflow/generics/task.py rename to taskflow/task.py diff --git a/taskflow/tests/unit/memory_api/__init__.py b/taskflow/tests/unit/memory_api/__init__.py index b33107da..44fa8d98 100644 --- a/taskflow/tests/unit/memory_api/__init__.py +++ b/taskflow/tests/unit/memory_api/__init__.py @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow.backends import api as b_api +from taskflow.persistence.backends import api as b_api def setUpModule(): diff --git a/taskflow/tests/unit/memory_api/test_flowdetail_api.py b/taskflow/tests/unit/memory_api/test_flowdetail_api.py index 38d07ea1..24fd0972 100644 --- a/taskflow/tests/unit/memory_api/test_flowdetail_api.py +++ b/taskflow/tests/unit/memory_api/test_flowdetail_api.py @@ -19,11 +19,11 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import flowdetail from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as flow +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import flowdetail from taskflow.tests import utils diff --git a/taskflow/tests/unit/memory_api/test_logbook_api.py b/taskflow/tests/unit/memory_api/test_logbook_api.py index 633844a4..136027ff 100644 --- a/taskflow/tests/unit/memory_api/test_logbook_api.py +++ b/taskflow/tests/unit/memory_api/test_logbook_api.py @@ -19,11 +19,11 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import logbook from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as flow +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import logbook from taskflow.tests import utils diff --git a/taskflow/tests/unit/memory_api/test_taskdetail_api.py b/taskflow/tests/unit/memory_api/test_taskdetail_api.py index ec3bc22d..bb49afc3 100644 --- a/taskflow/tests/unit/memory_api/test_taskdetail_api.py +++ b/taskflow/tests/unit/memory_api/test_taskdetail_api.py @@ -19,10 +19,10 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import taskdetail from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import taskdetail from taskflow.tests import utils diff --git a/taskflow/tests/unit/sql_db_api/__init__.py b/taskflow/tests/unit/sql_db_api/__init__.py index e4ad98fb..9ca097bb 100644 --- a/taskflow/tests/unit/sql_db_api/__init__.py +++ b/taskflow/tests/unit/sql_db_api/__init__.py @@ -20,8 +20,8 @@ import os from os import path -from taskflow.backends import api as b_api -from taskflow.backends.sqlalchemy import models +from taskflow.persistence.backends import api as b_api +from taskflow.persistence.backends.sqlalchemy import models def setUpModule(): diff --git a/taskflow/tests/unit/sql_db_api/test_flowdetail_api.py b/taskflow/tests/unit/sql_db_api/test_flowdetail_api.py index 38d07ea1..24fd0972 100644 --- a/taskflow/tests/unit/sql_db_api/test_flowdetail_api.py +++ b/taskflow/tests/unit/sql_db_api/test_flowdetail_api.py @@ -19,11 +19,11 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import flowdetail from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as flow +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import flowdetail from taskflow.tests import utils diff --git a/taskflow/tests/unit/sql_db_api/test_logbook_api.py b/taskflow/tests/unit/sql_db_api/test_logbook_api.py index 633844a4..136027ff 100644 --- a/taskflow/tests/unit/sql_db_api/test_logbook_api.py +++ b/taskflow/tests/unit/sql_db_api/test_logbook_api.py @@ -19,11 +19,11 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import logbook from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as flow +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import logbook from taskflow.tests import utils diff --git a/taskflow/tests/unit/sql_db_api/test_taskdetail_api.py b/taskflow/tests/unit/sql_db_api/test_taskdetail_api.py index ec3bc22d..bb49afc3 100644 --- a/taskflow/tests/unit/sql_db_api/test_taskdetail_api.py +++ b/taskflow/tests/unit/sql_db_api/test_taskdetail_api.py @@ -19,10 +19,10 @@ """Import required libraries""" import unittest2 -from taskflow.backends import api as b_api from taskflow import exceptions as exception -from taskflow.generics import taskdetail from taskflow.openstack.common import uuidutils +from taskflow.persistence.backends import api as b_api +from taskflow.persistence import taskdetail from taskflow.tests import utils diff --git a/taskflow/tests/unit/test_linear_flow.py b/taskflow/tests/unit/test_linear_flow.py index 99dfb6ef..29a862b6 100644 --- a/taskflow/tests/unit/test_linear_flow.py +++ b/taskflow/tests/unit/test_linear_flow.py @@ -22,9 +22,9 @@ from taskflow import decorators from taskflow import exceptions as exc from taskflow import states -from taskflow.backends import memory from taskflow.patterns import linear_flow as lw from taskflow.patterns.resumption import logbook as lr +from taskflow.persistence.backends import memory from taskflow.tests import utils diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index 869c9a7c..47de196d 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow.generics import task +from taskflow import task ARGS_KEY = '__args__' KWARGS_KEY = '__kwargs__' diff --git a/tools/pip-requires b/tools/pip-requires index f75a5c56..6ae0e393 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -3,8 +3,7 @@ oslo.config>=1.1.0 iso8601 # Very nice graph library networkx>=1.5 -# Only needed if celery/distributed flows used. -celery +six # Only needed if database backend used. sqlalchemy>=0.7,<=0.7.99 sqlalchemy-migrate>=0.7