debtcollector for globals

The deprecation shim created by Doug turns out to be rather useful.
It emits a warning when an global (attribute of a module) is
referenced but that global has been moved to another module.

This update makes the following changes to the shim:

 - Rename it to _MovedGlobals to better describe what it is a
   debtcollector for.

 - Use inspect to get the original reference and to check that
   _MovedGlobals is called from the last line of a module.

 - Save the old reference automatically in the instance to prevent it
   from getting garbage collected.

 - Beef up the _moved_global() method for moving/renaming individual
   globals, allowing it to move and rename or rename in place.

Change-Id: I868aa4a3129dd05467a103364088efbb86bc5d0f
This commit is contained in:
Henry Gessau 2016-08-06 23:53:13 -04:00 committed by Henry Gessau
parent 8eaa1808e5
commit 99cfd671be
15 changed files with 345 additions and 108 deletions

View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from debtcollector import moves
from neutron_lib.api import converters as lib_converters
from neutron_lib.api import validators as lib_validators
@ -29,7 +27,7 @@ from neutron.common import _deprecate
# Defining a constant to avoid repeating string literal in several modules
SHARED = 'shared'
_deprecate._DeprecateSubset.and_also('UNLIMITED', lib_validators)
_deprecate._moved_global('UNLIMITED', new_module=lib_validators)
# TODO(HenryG): use DB field sizes (neutron-lib 0.1.1)
NAME_MAX_LEN = 255
@ -104,9 +102,9 @@ convert_none_to_empty_dict = _lib('convert_none_to_empty_dict')
convert_to_list = _lib('convert_to_list')
_deprecate._DeprecateSubset.and_also('MAC_PATTERN', lib_validators)
_deprecate._moved_global('MAC_PATTERN', new_module=lib_validators)
_deprecate._DeprecateSubset.and_also('validators', lib_validators)
_deprecate._moved_global('validators', new_module=lib_validators)
# Define constants for base resource name
@ -467,16 +465,8 @@ def verify_attributes(res_dict, attr_info):
# ATTR_NOT_SPECIFIED
# HEX_ELEM
# UUID_PATTERN
# Neutron-lib migration shim. This will wrap any constants that are moved
# to that library in a deprecation warning, until they can be updated to
# import directly from their new location.
# If you're wondering why we bother saving _OLD_REF, it is because if we
# do not, then the original module we are overwriting gets garbage collected,
# and then you will find some super strange behavior with inherited classes
# and the like. Saving a ref keeps it around.
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), constants)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
#
# Neutron-lib migration shim. This will emit a deprecation warning on any
# reference to constants that have been moved out of this module and into
# the neutron_lib.constants module.
_deprecate._MovedGlobals(constants)

View File

@ -11,52 +11,157 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import debtcollector
"""
Provide a deprecation method for globals.
NOTE: This module may be a candidate for adoption by debtcollector.
"""
import inspect
import sys
import debtcollector
from neutron._i18n import _
class _DeprecateSubset(object):
additional = {}
class _MovedGlobals(object):
"""Override a module to deprecate moved globals.
def __init__(self, my_globals, other_mod):
self.other_mod = other_mod
self.my_globals = my_globals
This class is used when globals (attributes of a module) need to be
marked as deprecated. It can be used in either or both of two ways:
@classmethod
def and_also(cls, name, other_mod):
cls.additional[name] = other_mod
1. By specifying a default new module, all accesses to a global in
the source module will emit a warning if the global does not exist
in the source module and it does exist in the new module. This way
is intended to be used when many globals are moved from one module
to another.
2. By explicitly deprecating individual globals with the _moved_global()
function, see below.
This class must be called from the last line in a module, as follows:
``_deprecate._MovedGlobals(default_new_module)``
or
``_deprecate._MovedGlobals()``
Args:
:param default_new_module: The default new location for moved globals
:type default_new_module: module or None
Attributes:
:ivar _mg__my_globals: The current vars() of the source module
:type _mg__my_globals: dict
:ivar _mg__default_new_mod: The default location for moved globals
:type _mg__default_new_mod: module or None
:ivar _mg__old_ref: The original reference to the source module
:type _mg__old_ref: module
:cvar _mg__moves: Moves (and renames) not involving default_new_module
:type _mg__moves: dict
NOTE: An instance of _MovedGlobals overrides the module it is called
from, so instance and class variables appear in the module namespace.
To prevent collisions with existing globals, the instance and class
variable names here are prefixed with ``_mg__``.
"""
# Here we store individual moves and renames. This is a dict where
# key = (old_module, old_name)
# value = (new_module, new_name)
# If new_module is the same as old_module then it is a rename in place.
_mg__moves = {}
def __init__(self, default_new_module=None):
# To avoid infinite recursion at inspect.getsourcelines() below we
# must initialize self._mg__my_globals early here.
self._mg__my_globals = {}
self._mg__default_new_mod = default_new_module
caller_frame = inspect.stack()[1][0]
caller_line = inspect.getframeinfo(caller_frame).lineno
source_module = inspect.getmodule(caller_frame)
src_mod_last_line = len(inspect.getsourcelines(source_module)[0])
if caller_line < src_mod_last_line:
raise SystemExit(_("_MovedGlobals() not called from last "
"line in %s") % source_module.__file__)
self._mg__my_globals = vars(source_module)
# When we return from here we override the sys.modules[] entry
# for the source module with this instance. We must keep a
# reference to the original module to prevent it from being
# garbage collected.
self._mg__old_ref = source_module
sys.modules[source_module.__name__] = self
def __getattr__(self, name):
a = self.my_globals.get(name)
if not name.startswith("__") and not inspect.ismodule(a):
other_mod = self.additional.get(name) or self.other_mod
if name in vars(other_mod):
# These should be enabled after most have been cleaned up
# in neutron proper, which may not happen during the busy M-3.
value = self._mg__my_globals.get(name)
if not name.startswith("__") and not inspect.ismodule(value):
old_module = self._mg__old_ref
specified_move = self._mg__moves.get((old_module, name))
if specified_move:
new_module, new_name = specified_move
else:
new_module, new_name = self._mg__default_new_mod, name
if new_module and new_name in vars(new_module):
old_location = '%s.%s' % (old_module.__name__, name)
new_location = '%s.%s' % (new_module.__name__, new_name)
changed = 'renamed' if old_module == new_module else 'moved'
debtcollector.deprecate(
name,
message='moved to %s' % other_mod.__name__,
old_location,
message='%s to %s' % (changed, new_location),
stacklevel=4)
return vars(other_mod)[name]
return vars(new_module)[new_name]
try:
return self.my_globals[name]
return self._mg__my_globals[name]
except KeyError:
raise AttributeError(
_("'module' object has no attribute '%s'") % name)
def __setattr__(self, name, val):
if name in ('other_mod', 'my_globals'):
return super(_DeprecateSubset, self).__setattr__(name, val)
self.my_globals[name] = val
if name.startswith('_mg__'):
return super(_MovedGlobals, self).__setattr__(name, val)
self._mg__my_globals[name] = val
def __delattr__(self, name):
if name not in self.my_globals:
if name not in self._mg__my_globals:
raise AttributeError(
_("'module' object has no attribute '%s'") % name)
self.my_globals.pop(name)
self._mg__my_globals.pop(name)
def _moved_global(old_name, new_module=None, new_name=None):
"""Deprecate a single attribute in a module.
This function is used to move an attribute to a module that differs
from _mg__default_new_mod in _MovedGlobals. It also handles renames.
NOTE: This function has no effect if _MovedGlobals() is not called
at the end of the module containing the attribute.
[TODO(HenryG): Figure out a way of asserting on this.]
:param old_name: The name of the attribute that was moved/renamed.
:type old_name: str
:param new_module: The new module where the attribute is now.
:type new_module: module
:param new_name: The new name of the attribute.
:type new_name: str
"""
assert new_module or new_name # One or both must be new
if isinstance(new_module, _MovedGlobals):
# The new module has been shimmed, get the original
new_module = new_module._mg__old_ref
old_module = inspect.getmodule(inspect.stack()[1][0]) # caller's module
new_module = new_module or old_module
new_name = new_name or old_name
_MovedGlobals._mg__moves[(old_module, old_name)] = (new_module, new_name)

View File

@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from neutron_lib import constants as lib_constants
from neutron.common import _deprecate
@ -135,15 +133,8 @@ DVR_FIP_LL_CIDR = '169.254.64.0/18'
L3_HA_NET_CIDR = '169.254.192.0/18'
METADATA_CIDR = '169.254.169.254/32'
# Neutron-lib migration shim. This will wrap any constants that are moved
# to that library in a deprecation warning, until they can be updated to
# import directly from their new location.
# If you're wondering why we bother saving _OLD_REF, it is because if we
# do not, then the original module we are overwriting gets garbage collected,
# and then you will find some super strange behavior with inherited classes
# and the like. Saving a ref keeps it around.
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), lib_constants)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
# Neutron-lib migration shim. This will emit a deprecation warning on any
# reference to constants that have been moved out of this module and into
# the neutron_lib.constants module.
_deprecate._MovedGlobals(lib_constants)

View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from neutron_lib import exceptions as e
from neutron._i18n import _
@ -314,15 +312,7 @@ class TenantIdProjectIdFilterConflict(e.BadRequest):
message = _("Both tenant_id and project_id passed as filters.")
# Neutron-lib migration shim. This will wrap any exceptions that are moved
# to that library in a deprecation warning, until they can be updated to
# import directly from their new location.
# If you're wondering why we bother saving _OLD_REF, it is because if we
# do not, then the original module we are overwriting gets garbage collected,
# and then you will find some super strange behavior with inherited classes
# and the like. Saving a ref keeps it around.
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), e)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
# Neutron-lib migration shim. This will emit a deprecation warning on any
# reference to exceptions that have been moved out of this module and into
# the neutron_lib.exceptions module.
_deprecate._MovedGlobals(e)

View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from neutron_lib import constants
from oslo_utils import uuidutils
from sqlalchemy.orm import exc
@ -27,6 +25,9 @@ from neutron.extensions import address_scope as ext_address_scope
from neutron.objects import subnetpool as subnetpool_obj
_deprecate._moved_global('AddressScope', new_module=address_scope_model)
class AddressScopeDbMixin(ext_address_scope.AddressScopePluginBase):
"""Mixin class to add address scope to db_base_plugin_v2."""
@ -144,8 +145,4 @@ class AddressScopeDbMixin(ext_address_scope.AddressScopePluginBase):
attr.NETWORKS, ['_extend_network_dict_address_scope'])
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(),
address_scope_model)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._MovedGlobals()

View File

@ -10,13 +10,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from neutron.common import _deprecate
from neutron.db.models import allowed_address_pair as aap_models
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), aap_models)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._moved_global('AllowedAddressPair', new_module=aap_models)
_deprecate._MovedGlobals()

View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
import netaddr
from neutron_lib.api import validators
from neutron_lib import constants
@ -36,6 +34,11 @@ from neutron.db.models import securitygroup as sg_models
from neutron.extensions import securitygroup as ext_sg
_deprecate._moved_global('DefaultSecurityGroup', new_module=sg_models)
_deprecate._moved_global('SecurityGroupPortBinding', new_module=sg_models)
_deprecate._moved_global('SecurityGroupRule', new_module=sg_models)
class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase):
"""Mixin class to add security group to db_base_plugin_v2."""
@ -743,7 +746,4 @@ class SecurityGroupDbMixin(ext_sg.SecurityGroupPluginBase):
return need_notify
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), sg_models)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._MovedGlobals()

View File

@ -16,14 +16,15 @@
# TODO(ihrachys): consider renaming the module since now it does not contain
# any models at all
import sys
from neutron.api.v2 import attributes
from neutron.common import _deprecate
from neutron.db import common_db_mixin
from neutron.db.models import subnet_service_type as sst_model
_deprecate._moved_global('SubnetServiceType', new_module=sst_model)
class SubnetServiceTypeMixin(object):
"""Mixin class to extend subnet with service type attribute"""
@ -36,7 +37,4 @@ class SubnetServiceTypeMixin(object):
attributes.SUBNETS, [_extend_subnet_service_types])
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), sst_model)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._MovedGlobals()

View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from neutron_lib import exceptions as exc
from oslo_config import cfg
from oslo_db import exception as db_exc
@ -43,6 +41,9 @@ flat_opts = [
cfg.CONF.register_opts(flat_opts, "ml2_type_flat")
_deprecate._moved_global('FlatAllocation', new_module=type_flat_model)
class FlatTypeDriver(helpers.BaseTypeDriver):
"""Manage state for flat networks with ML2.
@ -139,7 +140,4 @@ class FlatTypeDriver(helpers.BaseTypeDriver):
return min(mtu) if mtu else 0
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), type_flat_model)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._MovedGlobals()

View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from neutron_lib import exceptions as n_exc
from oslo_config import cfg
from oslo_log import log
@ -38,6 +36,10 @@ gre_opts = [
cfg.CONF.register_opts(gre_opts, "ml2_type_gre")
_deprecate._moved_global('GreAllocation', new_module=gre_model)
_deprecate._moved_global('GreEndpoints', new_module=gre_model)
class GreTypeDriver(type_tunnel.EndpointTunnelTypeDriver):
def __init__(self):
@ -70,7 +72,4 @@ class GreTypeDriver(type_tunnel.EndpointTunnelTypeDriver):
return mtu - p_const.GRE_ENCAP_OVERHEAD if mtu else 0
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_OLD_REF = sys.modules[__name__]
sys.modules[__name__] = _deprecate._DeprecateSubset(globals(), gre_model)
# WARNING: THESE MUST BE THE LAST TWO LINES IN THIS MODULE
_deprecate._MovedGlobals()

View File

@ -0,0 +1,32 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Used by test cases in test__deprecate.py
"""
from neutron.common import _deprecate
from neutron.tests.unit.common import moved_globals_target
# a has been moved to moved_globals_target.a
b = 'barasingha'
# c has been renamed to d
d = 'capybara'
# e has been moved to moved_globals_target.f
g = 'gelada'
_deprecate._moved_global('c', new_name='d')
_deprecate._moved_global('e', new_name='f', new_module=moved_globals_target)
_deprecate._MovedGlobals(moved_globals_target)

View File

@ -0,0 +1,25 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Used by test cases in test__deprecate.py
"""
from neutron.common import _deprecate
from neutron.tests.unit.common import moved_globals_target
global1 = 'foo'
_deprecate._MovedGlobals(moved_globals_target)
global2 = 'bar'

View File

@ -0,0 +1,19 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Used by test cases in test__deprecate.py
"""
a = 'aardvark'
f = 'echidna'

View File

@ -0,0 +1,95 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from oslo_utils import importutils
from neutron.tests import base
from neutron.tests.unit.common import moved_globals_target as new_mod
def module_path(code):
return 'neutron.tests.unit.common.moved_globals_' + code
def import_code(code):
return importutils.import_module(module_path(code))
def expect_moved(code, name, new_name=None):
old_path = '.'.join([module_path(code), name])
new_path = '.'.join([new_mod.__name__, new_name or name])
message = 'moved to ' + new_path
return old_path, message
def expect_renamed(code, old_name, new_name):
old_path = '.'.join([module_path(code), old_name])
new_path = '.'.join([module_path(code), new_name])
message = 'renamed to ' + new_path
return old_path, message
class TestMovedGlobals(base.BaseTestCase):
def test_moved_global(self):
code = 'code1'
old_mod = import_code(code)
with mock.patch('debtcollector.deprecate') as dc:
self.assertEqual(new_mod.a, old_mod.a)
old_path, msg = expect_moved(code, 'a')
dc.assert_called_once_with(old_path, message=msg, stacklevel=4)
def test_moved_global_no_attr(self):
mod = import_code('code1')
self.assertRaises(AttributeError, lambda: mod.NO_SUCH_ATTRIBUTE)
def test_renamed_global(self):
code = 'code1'
mod = import_code(code)
with mock.patch('debtcollector.deprecate') as dc:
self.assertEqual(mod.d, mod.c)
old_path, msg = expect_renamed(code, 'c', 'd')
dc.assert_called_once_with(old_path, message=msg, stacklevel=4)
def test_moved_global_renamed(self):
code = 'code1'
old_mod = import_code(code)
with mock.patch('debtcollector.deprecate') as dc:
self.assertEqual(new_mod.f, old_mod.e)
old_path, msg = expect_moved(code, 'e', new_name='f')
dc.assert_called_once_with(old_path, message=msg, stacklevel=4)
def test_set_unmoved_global(self):
mod = import_code('code1')
mod.d = 'dibatag'
self.assertEqual('dibatag', mod.d)
def test_set_new_global(self):
mod = import_code('code1')
mod.n = 'nyala'
self.assertEqual('nyala', mod.n)
def test_delete_unmoved_global(self):
mod = import_code('code1')
self.assertEqual('gelada', mod.g)
def delete_g():
del mod.g
delete_g()
self.assertRaises(AttributeError, lambda: mod.g)
self.failUnlessRaises(AttributeError, delete_g)
def test_not_last_line(self):
self.assertRaises(SystemExit, import_code, 'code2')

View File

@ -26,7 +26,7 @@ from neutron.common import exceptions as nexc
def check_globals(things, nmod, lmod):
core = vars(nmod)['my_globals']
core = vars(nmod)['_mg__my_globals']
lib = vars(lmod)
moved_things = []
for thing in core: