From 4e514f41e57983e728db9025126df6f791a2594a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Dec 2014 18:58:18 -0800 Subject: [PATCH] Move over to using oslo.utils [reflection, uuidutils] The reflection module is now part of oslo.utils so we should remove our local version and use that version instead; this also goes for the uuidutils module which is now part of oslo.utils as well so we no longer need our local version copied from the incubator... Note that one reflection method `find_subclasses` which was to specific to taskflow is now moved to the misc utility module instead of its prior home in the reflection module. Change-Id: I069881c80b0b2916cc0c414992b80171f7eeb79f --- doc/source/utils.rst | 5 - openstack-common.conf | 1 - taskflow/atom.py | 2 +- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/helpers.py | 2 +- taskflow/engines/worker_based/endpoint.py | 3 +- taskflow/engines/worker_based/executor.py | 2 +- taskflow/engines/worker_based/protocol.py | 2 +- taskflow/engines/worker_based/worker.py | 5 +- taskflow/examples/create_parallel_volume.py | 3 +- taskflow/examples/fake_billing.py | 2 +- taskflow/examples/resume_vm_boot.py | 3 +- taskflow/flow.py | 3 +- taskflow/jobs/backends/impl_zookeeper.py | 2 +- taskflow/jobs/job.py | 3 +- taskflow/openstack/__init__.py | 0 taskflow/openstack/common/__init__.py | 17 -- taskflow/openstack/common/uuidutils.py | 37 --- .../persistence/backends/sqlalchemy/models.py | 2 +- taskflow/persistence/logbook.py | 2 +- taskflow/storage.py | 4 +- taskflow/task.py | 2 +- taskflow/tests/unit/jobs/base.py | 2 +- taskflow/tests/unit/jobs/test_zk_job.py | 2 +- taskflow/tests/unit/persistence/base.py | 3 +- .../unit/persistence/test_zk_persistence.py | 2 +- taskflow/tests/unit/test_listeners.py | 2 +- taskflow/tests/unit/test_storage.py | 3 +- taskflow/tests/unit/test_utils.py | 255 ------------------ .../tests/unit/worker_based/test_endpoint.py | 3 +- .../unit/worker_based/test_message_pump.py | 3 +- .../tests/unit/worker_based/test_pipeline.py | 2 +- .../tests/unit/worker_based/test_protocol.py | 2 +- .../tests/unit/worker_based/test_worker.py | 2 +- taskflow/types/cache.py | 3 +- taskflow/types/failure.py | 2 +- taskflow/types/notifier.py | 3 +- taskflow/utils/deprecation.py | 3 +- taskflow/utils/kazoo_utils.py | 2 +- taskflow/utils/misc.py | 48 +++- taskflow/utils/persistence_utils.py | 2 +- taskflow/utils/reflection.py | 251 ----------------- 42 files changed, 91 insertions(+), 608 deletions(-) delete mode 100644 taskflow/openstack/__init__.py delete mode 100644 taskflow/openstack/common/__init__.py delete mode 100644 taskflow/openstack/common/uuidutils.py delete mode 100644 taskflow/utils/reflection.py diff --git a/doc/source/utils.rst b/doc/source/utils.rst index 968c2c048..8125aac6f 100644 --- a/doc/source/utils.rst +++ b/doc/source/utils.rst @@ -38,11 +38,6 @@ Persistence .. automodule:: taskflow.utils.persistence_utils -Reflection -~~~~~~~~~~ - -.. automodule:: taskflow.utils.reflection - Threading ~~~~~~~~~ diff --git a/openstack-common.conf b/openstack-common.conf index 9db6be0aa..127bc8399 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,6 @@ [DEFAULT] # The list of modules to copy from oslo-incubator.git -module=uuidutils script=tools/run_cross_tests.sh # The base module to hold the copy of openstack.common diff --git a/taskflow/atom.py b/taskflow/atom.py index 2cd665ac9..3131664fc 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -15,11 +15,11 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection import six from taskflow import exceptions from taskflow.utils import misc -from taskflow.utils import reflection def _save_as_to_mapping(save_as): diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index ffc3a80a8..0f2a8a204 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -18,6 +18,7 @@ import contextlib import threading from oslo.utils import excutils +from oslo.utils import reflection from taskflow.engines.action_engine import compiler from taskflow.engines.action_engine import executor @@ -29,7 +30,6 @@ from taskflow import storage as atom_storage from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection @contextlib.contextmanager diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 1943aa735..2c84a3c30 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -19,6 +19,7 @@ import itertools import traceback from oslo.utils import importutils +from oslo.utils import reflection import six import stevedore.driver @@ -28,7 +29,6 @@ from taskflow.persistence import backends as p_backends from taskflow.utils import deprecation from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils -from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/endpoint.py b/taskflow/engines/worker_based/endpoint.py index 58637e11d..0f883adec 100644 --- a/taskflow/engines/worker_based/endpoint.py +++ b/taskflow/engines/worker_based/endpoint.py @@ -14,8 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection + from taskflow.engines.action_engine import executor -from taskflow.utils import reflection class Endpoint(object): diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index bdef7bffb..450f493fb 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -17,6 +17,7 @@ import functools import threading +from oslo.utils import reflection from oslo.utils import timeutils from taskflow.engines.action_engine import executor @@ -28,7 +29,6 @@ from taskflow import logging from taskflow import task as task_atom from taskflow.types import timing as tt from taskflow.utils import misc -from taskflow.utils import reflection from taskflow.utils import threading_utils as tu LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index e2b40e601..96ba84c40 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -20,6 +20,7 @@ import threading from concurrent import futures import jsonschema from jsonschema import exceptions as schema_exc +from oslo.utils import reflection from oslo.utils import timeutils import six @@ -29,7 +30,6 @@ from taskflow import logging from taskflow.types import failure as ft from taskflow.types import timing as tt from taskflow.utils import lock_utils -from taskflow.utils import reflection # NOTE(skudriashev): This is protocol states and events, which are not # related to task states. diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 18627e27c..f75b7a8d5 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -21,12 +21,13 @@ import string import sys from concurrent import futures +from oslo.utils import reflection from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server from taskflow import logging from taskflow import task as t_task -from taskflow.utils import reflection +from taskflow.utils import misc from taskflow.utils import threading_utils as tu from taskflow import version @@ -103,7 +104,7 @@ class Worker(object): @staticmethod def _derive_endpoints(tasks): """Derive endpoints from list of strings, classes or packages.""" - derived_tasks = reflection.find_subclasses(tasks, t_task.BaseTask) + derived_tasks = misc.find_subclasses(tasks, t_task.BaseTask) return [endpoint.Endpoint(task) for task in derived_tasks] def _generate_banner(self): diff --git a/taskflow/examples/create_parallel_volume.py b/taskflow/examples/create_parallel_volume.py index 5185330b4..0416a25ee 100644 --- a/taskflow/examples/create_parallel_volume.py +++ b/taskflow/examples/create_parallel_volume.py @@ -28,11 +28,12 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) +from oslo.utils import reflection + from taskflow import engines from taskflow.listeners import printing from taskflow.patterns import unordered_flow as uf from taskflow import task -from taskflow.utils import reflection # INTRO: This examples shows how unordered_flow can be used to create a large # number of fake volumes in parallel (or serially, depending on a constant that diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index 22c75cd91..0fbe81f7e 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -27,10 +27,10 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) +from oslo.utils import uuidutils from taskflow import engines from taskflow.listeners import printing -from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index 514f3336d..f400d0d1e 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -31,9 +31,10 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), sys.path.insert(0, top_dir) sys.path.insert(0, self_dir) +from oslo.utils import uuidutils + from taskflow import engines from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task diff --git a/taskflow/flow.py b/taskflow/flow.py index c5fb42960..683cf8b49 100644 --- a/taskflow/flow.py +++ b/taskflow/flow.py @@ -16,10 +16,9 @@ import abc +from oslo.utils import reflection import six -from taskflow.utils import reflection - # Link metadata keys that have inherent/special meaning. # # This key denotes the link is an invariant that ensures the order is diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index e25ac51df..65f889021 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -25,13 +25,13 @@ from kazoo.protocol import paths as k_paths from kazoo.recipe import watchers from oslo.serialization import jsonutils from oslo.utils import excutils +from oslo.utils import uuidutils import six from taskflow import exceptions as excp from taskflow.jobs import job as base_job from taskflow.jobs import jobboard from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import timing as tt from taskflow.utils import kazoo_utils diff --git a/taskflow/jobs/job.py b/taskflow/jobs/job.py index 41ac4c16e..23e33ee77 100644 --- a/taskflow/jobs/job.py +++ b/taskflow/jobs/job.py @@ -17,10 +17,9 @@ import abc +from oslo.utils import uuidutils import six -from taskflow.openstack.common import uuidutils - @six.add_metaclass(abc.ABCMeta) class Job(object): diff --git a/taskflow/openstack/__init__.py b/taskflow/openstack/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/taskflow/openstack/common/__init__.py b/taskflow/openstack/common/__init__.py deleted file mode 100644 index d1223eaf7..000000000 --- a/taskflow/openstack/common/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox')) diff --git a/taskflow/openstack/common/uuidutils.py b/taskflow/openstack/common/uuidutils.py deleted file mode 100644 index 234b880c9..000000000 --- a/taskflow/openstack/common/uuidutils.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2012 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False diff --git a/taskflow/persistence/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py index 47b8c8393..3f056de59 100644 --- a/taskflow/persistence/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -17,6 +17,7 @@ from oslo.serialization import jsonutils from oslo.utils import timeutils +from oslo.utils import uuidutils from sqlalchemy import Column, String, DateTime, Enum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ForeignKey @@ -24,7 +25,6 @@ from sqlalchemy.orm import backref from sqlalchemy.orm import relationship from sqlalchemy import types as types -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index ea6de4d00..6ae7d7c90 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -19,11 +19,11 @@ import abc import copy from oslo.utils import timeutils +from oslo.utils import uuidutils import six from taskflow import exceptions as exc from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import failure as ft diff --git a/taskflow/storage.py b/taskflow/storage.py index dcc51714a..e698ca4db 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -17,11 +17,12 @@ import abc import contextlib +from oslo.utils import reflection +from oslo.utils import uuidutils import six from taskflow import exceptions from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import retry from taskflow import states @@ -29,7 +30,6 @@ from taskflow import task from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection LOG = logging.getLogger(__name__) STATES_WITH_RESULTS = (states.SUCCESS, states.REVERTING, states.FAILURE) diff --git a/taskflow/task.py b/taskflow/task.py index fbee0296f..7a1c71809 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -18,13 +18,13 @@ import abc import copy +from oslo.utils import reflection import six from taskflow import atom from taskflow import logging from taskflow.types import notifier from taskflow.utils import misc -from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index 24da7d3ae..4c070f31b 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -18,9 +18,9 @@ import contextlib import time from kazoo.recipe import watchers +from oslo.utils import uuidutils from taskflow import exceptions as excp -from taskflow.openstack.common import uuidutils from taskflow.persistence.backends import impl_dir from taskflow import states from taskflow.test import mock diff --git a/taskflow/tests/unit/jobs/test_zk_job.py b/taskflow/tests/unit/jobs/test_zk_job.py index 5a536f9e2..8737a4f55 100644 --- a/taskflow/tests/unit/jobs/test_zk_job.py +++ b/taskflow/tests/unit/jobs/test_zk_job.py @@ -15,13 +15,13 @@ # under the License. from oslo.serialization import jsonutils +from oslo.utils import uuidutils import six import testtools from zake import fake_client from zake import utils as zake_utils from taskflow.jobs.backends import impl_zookeeper -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow import test from taskflow.tests.unit.jobs import base diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py index 50bb3b3f4..88660fd5a 100644 --- a/taskflow/tests/unit/persistence/base.py +++ b/taskflow/tests/unit/persistence/base.py @@ -16,8 +16,9 @@ import contextlib +from oslo.utils import uuidutils + from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states from taskflow.types import failure diff --git a/taskflow/tests/unit/persistence/test_zk_persistence.py b/taskflow/tests/unit/persistence/test_zk_persistence.py index 609de21fa..28463bb75 100644 --- a/taskflow/tests/unit/persistence/test_zk_persistence.py +++ b/taskflow/tests/unit/persistence/test_zk_persistence.py @@ -17,11 +17,11 @@ import contextlib from kazoo import exceptions as kazoo_exceptions +from oslo.utils import uuidutils import testtools from zake import fake_client from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.persistence import backends from taskflow.persistence.backends import impl_zookeeper from taskflow import test diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py index 210fe7982..c10bc282b 100644 --- a/taskflow/tests/unit/test_listeners.py +++ b/taskflow/tests/unit/test_listeners.py @@ -19,6 +19,7 @@ import logging import time from oslo.serialization import jsonutils +from oslo.utils import reflection import six from zake import fake_client @@ -37,7 +38,6 @@ from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.utils import misc from taskflow.utils import persistence_utils -from taskflow.utils import reflection from taskflow.utils import threading_utils diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index f774993ca..886e075a3 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -17,8 +17,9 @@ import contextlib import threading +from oslo.utils import uuidutils + from taskflow import exceptions -from taskflow.openstack.common import uuidutils from taskflow.persistence import backends from taskflow.persistence import logbook from taskflow import states diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 56e39199a..c69b769d1 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -19,266 +19,11 @@ import inspect import random import time -import six -import testtools - from taskflow import test -from taskflow.tests import utils as test_utils -from taskflow.types import failure -from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection from taskflow.utils import threading_utils -def mere_function(a, b): - pass - - -def function_with_defs(a, b, optional=None): - pass - - -def function_with_kwargs(a, b, **kwargs): - pass - - -class Class(object): - - def method(self, c, d): - pass - - @staticmethod - def static_method(e, f): - pass - - @classmethod - def class_method(cls, g, h): - pass - - -class CallableClass(object): - def __call__(self, i, j): - pass - - -class ClassWithInit(object): - def __init__(self, k, l): - pass - - -class CallbackEqualityTest(test.TestCase): - def test_different_simple_callbacks(self): - - def a(): - pass - - def b(): - pass - - self.assertFalse(reflection.is_same_callback(a, b)) - - def test_static_instance_callbacks(self): - - class A(object): - - @staticmethod - def b(a, b, c): - pass - - a = A() - b = A() - - self.assertTrue(reflection.is_same_callback(a.b, b.b)) - - def test_different_instance_callbacks(self): - - class A(object): - def b(self): - pass - - def __eq__(self, other): - return True - - b = A() - c = A() - - self.assertFalse(reflection.is_same_callback(b.b, c.b)) - self.assertTrue(reflection.is_same_callback(b.b, c.b, strict=False)) - - -class GetCallableNameTest(test.TestCase): - - def test_mere_function(self): - name = reflection.get_callable_name(mere_function) - self.assertEqual(name, '.'.join((__name__, 'mere_function'))) - - def test_method(self): - name = reflection.get_callable_name(Class.method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'method'))) - - def test_instance_method(self): - name = reflection.get_callable_name(Class().method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'method'))) - - def test_static_method(self): - name = reflection.get_callable_name(Class.static_method) - if six.PY3: - self.assertEqual(name, - '.'.join((__name__, 'Class', 'static_method'))) - else: - # NOTE(imelnikov): static method are just functions, class name - # is not recorded anywhere in them. - self.assertEqual(name, - '.'.join((__name__, 'static_method'))) - - def test_class_method(self): - name = reflection.get_callable_name(Class.class_method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'class_method'))) - - def test_constructor(self): - name = reflection.get_callable_name(Class) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_callable_class(self): - name = reflection.get_callable_name(CallableClass()) - self.assertEqual(name, '.'.join((__name__, 'CallableClass'))) - - def test_callable_class_call(self): - name = reflection.get_callable_name(CallableClass().__call__) - self.assertEqual(name, '.'.join((__name__, 'CallableClass', - '__call__'))) - - -# These extended/special case tests only work on python 3, due to python 2 -# being broken/incorrect with regard to these special cases... -@testtools.skipIf(not six.PY3, 'python 3.x is not currently available') -class GetCallableNameTestExtended(test.TestCase): - # Tests items in http://legacy.python.org/dev/peps/pep-3155/ - - class InnerCallableClass(object): - def __call__(self): - pass - - def test_inner_callable_class(self): - obj = self.InnerCallableClass() - name = reflection.get_callable_name(obj.__call__) - expected_name = '.'.join((__name__, 'GetCallableNameTestExtended', - 'InnerCallableClass', '__call__')) - self.assertEqual(expected_name, name) - - def test_inner_callable_function(self): - def a(): - - def b(): - pass - - return b - - name = reflection.get_callable_name(a()) - expected_name = '.'.join((__name__, 'GetCallableNameTestExtended', - 'test_inner_callable_function', '', - 'a', '', 'b')) - self.assertEqual(expected_name, name) - - def test_inner_class(self): - obj = self.InnerCallableClass() - name = reflection.get_callable_name(obj) - expected_name = '.'.join((__name__, - 'GetCallableNameTestExtended', - 'InnerCallableClass')) - self.assertEqual(expected_name, name) - - -class GetCallableArgsTest(test.TestCase): - - def test_mere_function(self): - result = reflection.get_callable_args(mere_function) - self.assertEqual(['a', 'b'], result) - - def test_function_with_defaults(self): - result = reflection.get_callable_args(function_with_defs) - self.assertEqual(['a', 'b', 'optional'], result) - - def test_required_only(self): - result = reflection.get_callable_args(function_with_defs, - required_only=True) - self.assertEqual(['a', 'b'], result) - - def test_method(self): - result = reflection.get_callable_args(Class.method) - self.assertEqual(['self', 'c', 'd'], result) - - def test_instance_method(self): - result = reflection.get_callable_args(Class().method) - self.assertEqual(['c', 'd'], result) - - def test_class_method(self): - result = reflection.get_callable_args(Class.class_method) - self.assertEqual(['g', 'h'], result) - - def test_class_constructor(self): - result = reflection.get_callable_args(ClassWithInit) - self.assertEqual(['k', 'l'], result) - - def test_class_with_call(self): - result = reflection.get_callable_args(CallableClass()) - self.assertEqual(['i', 'j'], result) - - def test_decorators_work(self): - @lock_utils.locked - def locked_fun(x, y): - pass - result = reflection.get_callable_args(locked_fun) - self.assertEqual(['x', 'y'], result) - - -class AcceptsKwargsTest(test.TestCase): - - def test_no_kwargs(self): - self.assertEqual( - reflection.accepts_kwargs(mere_function), False) - - def test_with_kwargs(self): - self.assertEqual( - reflection.accepts_kwargs(function_with_kwargs), True) - - -class GetClassNameTest(test.TestCase): - - def test_std_exception(self): - name = reflection.get_class_name(RuntimeError) - self.assertEqual(name, 'RuntimeError') - - def test_global_class(self): - name = reflection.get_class_name(failure.Failure) - self.assertEqual(name, 'taskflow.types.failure.Failure') - - def test_class(self): - name = reflection.get_class_name(Class) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_instance(self): - name = reflection.get_class_name(Class()) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_int(self): - name = reflection.get_class_name(42) - self.assertEqual(name, 'int') - - -class GetAllClassNamesTest(test.TestCase): - - def test_std_class(self): - names = list(reflection.get_all_class_names(RuntimeError)) - self.assertEqual(names, test_utils.RUNTIME_ERROR_CLASSES) - - def test_std_class_up_to(self): - names = list(reflection.get_all_class_names(RuntimeError, - up_to=Exception)) - self.assertEqual(names, test_utils.RUNTIME_ERROR_CLASSES[:-2]) - - class CachedPropertyTest(test.TestCase): def test_attribute_caching(self): diff --git a/taskflow/tests/unit/worker_based/test_endpoint.py b/taskflow/tests/unit/worker_based/test_endpoint.py index 36abb980d..53260b12c 100644 --- a/taskflow/tests/unit/worker_based/test_endpoint.py +++ b/taskflow/tests/unit/worker_based/test_endpoint.py @@ -14,11 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection + from taskflow.engines.worker_based import endpoint as ep from taskflow import task from taskflow import test from taskflow.tests import utils -from taskflow.utils import reflection class Task(task.Task): diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index cae4fa5c0..7b945a26f 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -14,9 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import uuidutils + from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index 2822a852d..53bf8f9ba 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -15,12 +15,12 @@ # under the License. from concurrent import futures +from oslo.utils import uuidutils from taskflow.engines.action_engine import executor as base_executor from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import executor as worker_executor from taskflow.engines.worker_based import server as worker_server -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index d2f2cc02e..4c34ed607 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -16,11 +16,11 @@ from concurrent import futures from oslo.utils import timeutils +from oslo.utils import uuidutils from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils from taskflow.types import failure diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index ff049a640..8fc76eb46 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection import six from taskflow.engines.worker_based import endpoint @@ -21,7 +22,6 @@ from taskflow.engines.worker_based import worker from taskflow import test from taskflow.test import mock from taskflow.tests import utils -from taskflow.utils import reflection class TestWorker(test.MockTestCase): diff --git a/taskflow/types/cache.py b/taskflow/types/cache.py index 61511e12e..c3ac7a18f 100644 --- a/taskflow/types/cache.py +++ b/taskflow/types/cache.py @@ -16,10 +16,9 @@ import threading +from oslo.utils import reflection import six -from taskflow.utils import reflection - class ExpiringCache(object): """Represents a thread-safe time-based expiring cache. diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index b9d7a399f..33fb345d4 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -18,10 +18,10 @@ import copy import sys import traceback +from oslo.utils import reflection import six from taskflow import exceptions as exc -from taskflow.utils import reflection def _copy_exc_info(exc_info): diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index 98511fba8..838585a2b 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -19,10 +19,9 @@ import contextlib import copy import logging +from oslo.utils import reflection import six -from taskflow.utils import reflection - LOG = logging.getLogger(__name__) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index f70d76703..5c4666623 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -17,10 +17,9 @@ import functools import warnings +from oslo.utils import reflection import six -from taskflow.utils import reflection - _CLASS_MOVED_PREFIX_TPL = "Class '%s' has moved to '%s'" _KIND_MOVED_PREFIX_TPL = "%s '%s' has moved to '%s'" _KWARG_MOVED_POSTFIX_TPL = ", please use the '%s' argument instead" diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index 0a9922bb4..ab449635f 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -16,11 +16,11 @@ from kazoo import client from kazoo import exceptions as k_exc +from oslo.utils import reflection import six from six.moves import zip as compat_zip from taskflow import exceptions as exc -from taskflow.utils import reflection def _parse_hosts(hosts): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 34910064a..a6b04fa71 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -23,9 +23,12 @@ import os import re import sys import threading +import types from oslo.serialization import jsonutils +from oslo.utils import importutils from oslo.utils import netutils +from oslo.utils import reflection import six from six.moves import map as compat_map from six.moves import range as compat_range @@ -34,7 +37,6 @@ from six.moves.urllib import parse as urlparse from taskflow.types import failure from taskflow.types import notifier from taskflow.utils import deprecation -from taskflow.utils import reflection NUMERIC_TYPES = six.integer_types + (float,) @@ -83,6 +85,50 @@ def merge_uri(uri, conf): return conf +def find_subclasses(locations, base_cls, exclude_hidden=True): + """Finds subclass types in the given locations. + + This will examines the given locations for types which are subclasses of + the base class type provided and returns the found subclasses (or fails + with exceptions if this introspection can not be accomplished). + + If a string is provided as one of the locations it will be imported and + examined if it is a subclass of the base class. If a module is given, + all of its members will be examined for attributes which are subclasses of + the base class. If a type itself is given it will be examined for being a + subclass of the base class. + """ + derived = set() + for item in locations: + module = None + if isinstance(item, six.string_types): + try: + pkg, cls = item.split(':') + except ValueError: + module = importutils.import_module(item) + else: + obj = importutils.import_class('%s.%s' % (pkg, cls)) + if not reflection.is_subclass(obj, base_cls): + raise TypeError("Item %s is not a %s subclass" % + (item, base_cls)) + derived.add(obj) + elif isinstance(item, types.ModuleType): + module = item + elif reflection.is_subclass(item, base_cls): + derived.add(item) + else: + raise TypeError("Item %s unexpected type: %s" % + (item, type(item))) + # If it's a module derive objects from it if we can. + if module is not None: + for (name, obj) in inspect.getmembers(module): + if name.startswith("_") and exclude_hidden: + continue + if reflection.is_subclass(obj, base_cls): + derived.add(obj) + return derived + + def parse_uri(uri): """Parses a uri into its components.""" # Do some basic validation before continuing... diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index 340f558a6..b8a153514 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -17,9 +17,9 @@ import contextlib from oslo.utils import timeutils +from oslo.utils import uuidutils from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow.utils import misc diff --git a/taskflow/utils/reflection.py b/taskflow/utils/reflection.py deleted file mode 100644 index 08eaf6c97..000000000 --- a/taskflow/utils/reflection.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import inspect -import types - -from oslo.utils import importutils -import six - -try: - _TYPE_TYPE = types.TypeType -except AttributeError: - _TYPE_TYPE = type - -# See: https://docs.python.org/2/library/__builtin__.html#module-__builtin__ -# and see https://docs.python.org/2/reference/executionmodel.html (and likely -# others)... -_BUILTIN_MODULES = ('builtins', '__builtin__', 'exceptions') - - -def _get_members(obj, exclude_hidden): - """Yields the members of an object, filtering by hidden/not hidden.""" - for (name, value) in inspect.getmembers(obj): - if name.startswith("_") and exclude_hidden: - continue - yield (name, value) - - -def find_subclasses(locations, base_cls, exclude_hidden=True): - """Finds subclass types in the given locations. - - This will examines the given locations for types which are subclasses of - the base class type provided and returns the found subclasses (or fails - with exceptions if this introspection can not be accomplished). - - If a string is provided as one of the locations it will be imported and - examined if it is a subclass of the base class. If a module is given, - all of its members will be examined for attributes which are subclasses of - the base class. If a type itself is given it will be examined for being a - subclass of the base class. - """ - derived = set() - for item in locations: - module = None - if isinstance(item, six.string_types): - try: - pkg, cls = item.split(':') - except ValueError: - module = importutils.import_module(item) - else: - obj = importutils.import_class('%s.%s' % (pkg, cls)) - if not is_subclass(obj, base_cls): - raise TypeError("Item %s is not a %s subclass" % - (item, base_cls)) - derived.add(obj) - elif isinstance(item, types.ModuleType): - module = item - elif is_subclass(item, base_cls): - derived.add(item) - else: - raise TypeError("Item %s unexpected type: %s" % - (item, type(item))) - # If it's a module derive objects from it if we can. - if module is not None: - for (_name, obj) in _get_members(module, exclude_hidden): - if is_subclass(obj, base_cls): - derived.add(obj) - return derived - - -def get_member_names(obj, exclude_hidden=True): - """Get all the member names for a object.""" - return [name for (name, _obj) in _get_members(obj, exclude_hidden)] - - -def get_class_name(obj, fully_qualified=True): - """Get class name for object. - - If object is a type, fully qualified name of the type is returned. - Else, fully qualified name of the type of the object is returned. - For builtin types, just name is returned. - """ - if not isinstance(obj, six.class_types): - obj = type(obj) - try: - built_in = obj.__module__ in _BUILTIN_MODULES - except AttributeError: - pass - else: - if built_in: - try: - return obj.__qualname__ - except AttributeError: - return obj.__name__ - pieces = [] - try: - pieces.append(obj.__qualname__) - except AttributeError: - pieces.append(obj.__name__) - if fully_qualified: - try: - pieces.insert(0, obj.__module__) - except AttributeError: - pass - return '.'.join(pieces) - - -def get_all_class_names(obj, up_to=object): - """Get class names of object parent classes. - - Iterate over all class names object is instance or subclass of, - in order of method resolution (mro). If up_to parameter is provided, - only name of classes that are sublcasses to that class are returned. - """ - if not isinstance(obj, six.class_types): - obj = type(obj) - for cls in obj.mro(): - if issubclass(cls, up_to): - yield get_class_name(cls) - - -def get_callable_name(function): - """Generate a name from callable. - - Tries to do the best to guess fully qualified callable name. - """ - method_self = get_method_self(function) - if method_self is not None: - # This is a bound method. - if isinstance(method_self, six.class_types): - # This is a bound class method. - im_class = method_self - else: - im_class = type(method_self) - try: - parts = (im_class.__module__, function.__qualname__) - except AttributeError: - parts = (im_class.__module__, im_class.__name__, function.__name__) - elif inspect.ismethod(function) or inspect.isfunction(function): - # This could be a function, a static method, a unbound method... - try: - parts = (function.__module__, function.__qualname__) - except AttributeError: - if hasattr(function, 'im_class'): - # This is a unbound method, which exists only in python 2.x - im_class = function.im_class - parts = (im_class.__module__, - im_class.__name__, function.__name__) - else: - parts = (function.__module__, function.__name__) - else: - im_class = type(function) - if im_class is _TYPE_TYPE: - im_class = function - try: - parts = (im_class.__module__, im_class.__qualname__) - except AttributeError: - parts = (im_class.__module__, im_class.__name__) - return '.'.join(parts) - - -def get_method_self(method): - if not inspect.ismethod(method): - return None - try: - return six.get_method_self(method) - except AttributeError: - return None - - -def is_same_callback(callback1, callback2, strict=True): - """Returns if the two callbacks are the same.""" - if callback1 is callback2: - # This happens when plain methods are given (or static/non-bound - # methods). - return True - if callback1 == callback2: - if not strict: - return True - # Two bound methods are equal if functions themselves are equal and - # objects they are applied to are equal. This means that a bound - # method could be the same bound method on another object if the - # objects have __eq__ methods that return true (when in fact it is a - # different bound method). Python u so crazy! - try: - self1 = six.get_method_self(callback1) - self2 = six.get_method_self(callback2) - return self1 is self2 - except AttributeError: - pass - return False - - -def is_bound_method(method): - """Returns if the given method is bound to an object.""" - return bool(get_method_self(method)) - - -def is_subclass(obj, cls): - """Returns if the object is class and it is subclass of a given class.""" - return inspect.isclass(obj) and issubclass(obj, cls) - - -def _get_arg_spec(function): - if isinstance(function, type): - bound = True - function = function.__init__ - elif isinstance(function, (types.FunctionType, types.MethodType)): - bound = is_bound_method(function) - function = getattr(function, '__wrapped__', function) - else: - function = function.__call__ - bound = is_bound_method(function) - return inspect.getargspec(function), bound - - -def get_callable_args(function, required_only=False): - """Get names of callable arguments. - - Special arguments (like ``*args`` and ``**kwargs``) are not included into - output. - - If required_only is True, optional arguments (with default values) - are not included into output. - """ - argspec, bound = _get_arg_spec(function) - f_args = argspec.args - if required_only and argspec.defaults: - f_args = f_args[:-len(argspec.defaults)] - if bound: - f_args = f_args[1:] - return f_args - - -def accepts_kwargs(function): - """Returns True if function accepts kwargs.""" - argspec, _bound = _get_arg_spec(function) - return bool(argspec.keywords)