From 50a557623b53044bc416a02be02bdbf168757af7 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 24 Mar 2015 21:30:24 +1100 Subject: [PATCH 1/8] Increment version to 1.10.5. --- docs/conf.py | 2 +- setup.py | 2 +- src/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6fca287..92d77c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ copyright = u'2013-2014, Graham Dumpleton' # The short X.Y version. version = '1.10' # The full version, including alpha/beta/rc tags. -release = '1.10.4' +release = '1.10.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index acb36d7..2b72705 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ class optional_build_ext(build_ext): setup_kwargs = dict( name = 'wrapt', - version = '1.10.4', + version = '1.10.5', description = 'Module for decorators, wrappers and monkey patching.', author = 'Graham Dumpleton', author_email = 'Graham.Dumpleton@gmail.com', diff --git a/src/__init__.py b/src/__init__.py index 1a989ac..05a8efa 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,4 +1,4 @@ -__version_info__ = ('1', '10', '4') +__version_info__ = ('1', '10', '5') __version__ = '.'.join(__version_info__) from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper, From 947dfd82ed543859ecb1691c4403f39cb2348218 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 24 Mar 2015 21:31:06 +1100 Subject: [PATCH 2/8] Update copyright years. --- LICENSE | 2 +- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index b82c72b..f92424f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013, Graham Dumpleton +Copyright (c) 2013-2015, Graham Dumpleton All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/docs/conf.py b/docs/conf.py index 92d77c1..b9849fe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,7 +41,7 @@ master_doc = 'index' # General information about the project. project = u'wrapt' -copyright = u'2013-2014, Graham Dumpleton' +copyright = u'2013-2015, Graham Dumpleton' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From b820fd2efb6056baa0e0e4100e1a3b2d68597a62 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 24 Mar 2015 21:31:47 +1100 Subject: [PATCH 3/8] Fix issue where multiple target modules registered in entry point for discovered module. --- docs/changes.rst | 9 +++++++++ src/importer.py | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 5271a61..7673e33 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,15 @@ Release Notes ============= +Version 1.10.5 +-------------- + +**Bugs Fixed** + +* Post import hook discovery was not working correctly where multiple + target modules were registered in the same entry point list. Only the + callback for the last would be called regardless of the target module. + Version 1.10.4 -------------- diff --git a/src/importer.py b/src/importer.py index b52d347..1f1e379 100644 --- a/src/importer.py +++ b/src/importer.py @@ -74,6 +74,15 @@ def register_post_import_hook(hook, name): # Register post import hooks defined as package entry points. +def _create_import_hook_from_entrypoint(entrypoint): + def import_hook(module): + __import__(entrypoint.module_name) + callback = sys.modules[entrypoint.module_name] + for attr in entrypoint.attrs: + callback = getattr(callback, attr) + return callback(module) + return import_hook + def discover_post_import_hooks(group): try: import pkg_resources @@ -81,14 +90,8 @@ def discover_post_import_hooks(group): return for entrypoint in pkg_resources.iter_entry_points(group=group): - def proxy_post_import_hook(module): - __import__(entrypoint.module_name) - callback = sys.modules[entrypoint.module_name] - for attr in entrypoint.attrs: - callback = getattr(callback, attr) - return callback(module) - - register_post_import_hook(proxy_post_import_hook, entrypoint.name) + callback = _create_import_hook_from_entrypoint(entrypoint) + register_post_import_hook(callback, entrypoint.name) # Indicate that a module has been loaded. Any post import hooks which # were registered against the target module will be invoked. If an From dcbb8b83f6b503b631d726715f25a9487639e1a2 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 24 Mar 2015 21:58:06 +1100 Subject: [PATCH 4/8] Extend register_post_import_hook() to allow deferred loading of import hook function by name. --- docs/changes.rst | 10 ++++++++++ src/importer.py | 27 ++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 7673e33..e6bf0d2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -10,6 +10,16 @@ Version 1.10.5 target modules were registered in the same entry point list. Only the callback for the last would be called regardless of the target module. +**Features Changed** + +* The ``register_post_import_hook()`` function, modelled after the + function of the same name in PEP-369 has been extended to allow a string + name to be supplied for the import hook. This needs to be of the form + ``module::function`` and will result in an import hook proxy being used + which will only load and call the function of the specified moduled when + the import hook is required. This avoids needing to load the code needed + to operate on the target module unless required. + Version 1.10.4 -------------- diff --git a/src/importer.py b/src/importer.py index 1f1e379..cb9d329 100644 --- a/src/importer.py +++ b/src/importer.py @@ -11,6 +11,9 @@ PY3 = sys.version_info[0] == 3 if PY3: import importlib + string_types = str, +else: + string_types = basestring, from .decorators import synchronized @@ -24,10 +27,32 @@ _post_import_hooks = {} _post_import_hooks_init = False _post_import_hooks_lock = threading.RLock() -# Register a new post import hook for the target module name. +# Register a new post import hook for the target module name. This +# differs from the PEP-369 implementation in that it also allows the +# hook function to be specified as a string consisting of the name of +# the callback in the form 'module:function'. This will result in a +# proxy callback being registered which will defer loading of the +# specified module containing the callback function until required. + +def _create_import_hook_from_string(name): + def import_hook(module): + module_name, function = name.split(':') + attrs = function.split('.') + __import__(module_name) + callback = sys.modules[module_name] + for attr in attrs: + callback = getattr(callback, attr) + return callback(module) + return import_hook @synchronized(_post_import_hooks_lock) def register_post_import_hook(hook, name): + # Create a deferred import hook if hook is a string name rather than + # a callable function. + + if isinstance(hook, string_types): + hook = _create_import_hook_from_string(hook) + # Automatically install the import hook finder if it has not already # been installed. From 077608c3aca359e03befc8f5ef294c58443d0ab5 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 14 Apr 2015 11:07:56 -0400 Subject: [PATCH 5/8] Add latest blog posts on monkey patching using wrapt. --- ...afely-applying-monkey-patches-in-python.md | 293 +++++++++++ ...ng-wrapt-to-support-testing-of-software.md | 458 ++++++++++++++++++ ...g-issues-when-monkey-patching-in-python.md | 294 +++++++++++ ...tomatic-patching-of-python-applications.md | 398 +++++++++++++++ blog/README.md | 11 + 5 files changed, 1454 insertions(+) create mode 100644 blog/11-safely-applying-monkey-patches-in-python.md create mode 100644 blog/12-using-wrapt-to-support-testing-of-software.md create mode 100644 blog/13-ordering-issues-when-monkey-patching-in-python.md create mode 100644 blog/14-automatic-patching-of-python-applications.md diff --git a/blog/11-safely-applying-monkey-patches-in-python.md b/blog/11-safely-applying-monkey-patches-in-python.md new file mode 100644 index 0000000..2924860 --- /dev/null +++ b/blog/11-safely-applying-monkey-patches-in-python.md @@ -0,0 +1,293 @@ +Safely applying monkey patches in Python +======================================== + +Monkey patching in Python is often see as being one of those things you +should never do. Some do regard it as a useful necessity you can't avoid in +order to patch bugs in third party code. Others will argue though that with +so much software being Open Source these days that you should simply submit +a fix to the upstream package maintainer. + +Monkey patching has its uses well beyond just patching bugs though. The two +most commonly used forms of monkey patching in Python which you might not +even equate with monkey patching are decorators and the use of mocking +libraries to assist in performing unit testing. Another not some common +case of monkey patching is to add instrumentation to existing Python code +in order to add performance monitoring capabilities. + +On the issue of decorators I wrote a quite detailed series of blog posts at +the start of last year about where decorators can cause problems. The +primary problem there was decorators which aren't implemented in a way +which preserve proper introspection capabilities, and which don't preserve +the correct semantics of the Python descriptor protocol when applied to +methods of classes. + +When one starts to talk about monkey patching arbitrary code, rather than +simply applying decorators to your own code, both of these issues become +even more important as you could quite easily interfere with the behaviour +of the existing code you are monkey patching in unexpected ways. + +This is especially the case when monkey patching methods of a class. This +is because when using decorators they would be applied while the class +definition is being constructed. When doing monkey patching you are coming +in after the class definition already exists and as a result you have to +deal with a number of non obvious problems. + +Now when I went and wrote the blog posts last year on decorators it was +effectively the result of what I learnt from implementing the wrapt +package. Although that package is known as providing a way for creating +well behaved decorators, that wasn't the primary aim in creating the +package. The real reason for creating the package was actually to implement +robust mechanisms for monkey patching code. It just so happened that the +same underlying principles and mechanism required to safely do monkey +patching apply to implementing the function wrappers required for +decorators. + +What I am going to do with this blog post is start to explain the monkey +patching capabilities of the wrapt package. + +Creating a decorator +-------------------- + +Before we jump into monkey patching of arbitrary code we first need to +recap how the wrapt package could be used to create a decorator. The +primary pattern for this was: + +``` +import wrapt +import inspect +@wrapt.decorator +def universal(wrapped, instance, args, kwargs): + if instance is None: + if inspect.isclass(wrapped): + # Decorator was applied to a class. + return wrapped(*args, **kwargs) + else: + # Decorator was applied to a function or staticmethod. + return wrapped(*args, **kwargs) + else: + if inspect.isclass(instance): + # Decorator was applied to a classmethod. + return wrapped(*args, **kwargs) + else: + # Decorator was applied to an instancemethod. + return wrapped(*args, **kwargs) +``` + +A special feature of the decorators that could be created by the wrapt +package was that within the decorator you could determine the context the +decorator was used in. That is, whether the decorator was applied to a +class, a function or static method, a class method or an instance method. + +For the case where the decorator was applied to an instance method you are +provided a separate argument to the instance of the class. For a class +method the separate argument is a reference to the class itself. In both +cases these are separated from the 'args' and 'kwargs' argument so you do +not need to fiddle around with extracting it yourself. + +A decorator created using wrapt is therefore what I call a universal +decorator. In other words, it is possible to create a single decorator +implementation that can be used across functions, methods and classes and +you can tell at the time of the call the scenario and adjust the behaviour +of the decorator accordingly. You no longer have to create multiple +implementations of a decorator and ensure that you are using the correct +one in each scenario. + +Using this decorator is then no different to any other way that decorators +would be used. + +``` +class Example(object): + + @universal + def name(self): + return 'name' +``` + +For those who have used Python long enough though, you would remember that +the syntax for applying a decorator in this way hasn't always existed. +Before the '@' syntax was allowed you could still create and use +decorators, but you had to be more explicit in applying them. That is, you +had to write: + +``` +class Example(object): + + def name(self): + return 'name' + name = universal(name) +``` + +This can still be done and when written this way it makes it clearer how +decorators are in a way a form of monkey patching. This is because often +all they are doing is introducing a wrapper around some existing function +which allows the call to the original function to be intercepted. The +wrapper function then allows you to perform actions either before or after +the call to the original function, or allow you to modify the arguments +passed to the wrapped function, or otherwise modify the result in some way, +or even substitute the result completely. + +What is an important distinction though with decorators is that the wrapper +function is being applied at the time the class containing the method is +being defined. In contrast more arbitrary monkey patching involves coming +in some time later after the class definition has been created and applying +the function wrapper at that point. + +In effect you are doing: + +``` +class Example(object): + def name(self): + return 'name' +Example.name = universal(Example.name) +``` + +Although a decorator function created using the wrapt package can be used +in this way and will still work as expected, in general I would discourage +this pattern for monkey patching an existing method of a class. + +This is because it isn't actually equivalent to doing the same thing within +the body of the class when it is defined. In particular the access of +'Example.name' actually invokes the descriptor protocol and so is returning +an instance method. We can see this by running the code: + +``` +class Example(object): + def name(self): + return 'name' + print type(name) +print type(Example.name) +``` + +which produces: + +``` + + +``` + +In general this may not matter, but I have seen some really strange corner +cases where the distinction has mattered. To deal with this therefore, the +wrapt package provides an alternate way of applying wrapper functions when +doing monkey patching after the fact. In the case of adding wrappers to +methods of class, this will use a mechanism which avoids any problems +caused by this subtle distinction. + +Adding function wrappers +------------------------ + +For general monkey patching using the wrapt package, rather than using the +decorator factory to create a decorator and then apply that to a function, +you instead define just the wrapper function and then use a separate +function to apply it to the target function. + +The prototype for the wrapper function is the same as before, but we simply +do not apply the '@wrapt.decorator' to it. + +``` +def wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) +``` + +To add the wrapper function to a target function we now use the +'wrapt.wrap_function_wrapper()' function. + +``` +class Example(object): + def name(self): + return 'name' +import wrapt + +wrapt.wrap_function_wrapper(Example, 'name', wrapper) +``` + +In this case we had the class in the same code file, but we could also have +done: + +``` +import example + +import wrapt +wrapt.wrap_function_wrapper(example, 'Example.name', wrapper) +``` + +That is, we provide the first argument as the module the target is defined +in, with the second argument being the object path to the method we wished +to apply the wrapper to. + +We could also skip importing the module altogether and just used the name +of the module. + +``` +import wrapt +wrapt.wrap_function_wrapper('example', 'Example.name', wrapper) +``` + +Just to prove that just about anything can be simplified by the user of a +decorator, we finally could write the whole thing as: + +``` +import wrapt + +@wrapt.patch_function_wrapper('example', 'Example.name') +def wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) +``` + +What will happen in this final example is that as soon as the module this +is contained in is imported, the specified target function defined in the +'example' module will automatically be monkey patched with the wrapper +function. + +Delayed patching is bad +----------------------- + +Now a very big warning is required at this point. Applying monkey patches +after the fact like this will not always work. + +The problem is that you are trying to apply a patch after the module has +been imported. In this case the 'wrapt.wrap_function_wrapper()' call will +ensure the module is imported if it wasn't already, but if the module had +already been imported previously by some other part of your code or by a +third party package you may have issues. + +In particular, it the target function you were trying to monkey patch was a +normal global function of the module, some other code could have grabbed a +direct reference to it by doing: + +``` +from example import function +``` + +If you come along later and have: + +``` +import wrapt +@wrapt.patch_function_wrapper('example', 'function') +def wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) +``` + +then yes the copy of the function contained in the target module will have +the wrapper applied, but the reference to it created by the other code will +not have the wrapper. + +To ensure that your wrapper is always used in this scenario you would need +to patch it not just in the original module, but in any modules where a +reference had been stored. This would only be practical in very limited +circumstances because in reality you are not going to have any idea where +the function might be getting used if it is a common function. + +This exact problem is one of the shortcomings in the way that monkey +patching is applied by packages such as gevent or eventlet. Both these +packages do delayed patching of functions and so are sensitive to the order +in which modules are imported. To get around this problem at least for +modules in the Python standard library, the 'time.sleep()' function which +they need to monkey patch, has to be patched not only in the 'time' module, +but also in the 'threading' module. + +There are some techniques one can use to try and avoid such problems but I +will defer explaining those to some time down the track. + +Instead for my next blog post I want to move onto some examples for where +monkey patching could be used by looking at how wrapt can be used as +alternative to packages such as the mock package when doing testing. diff --git a/blog/12-using-wrapt-to-support-testing-of-software.md b/blog/12-using-wrapt-to-support-testing-of-software.md new file mode 100644 index 0000000..00c3586 --- /dev/null +++ b/blog/12-using-wrapt-to-support-testing-of-software.md @@ -0,0 +1,458 @@ +Using wrapt to support testing of software +========================================== + +When talking about unit testing in Python, one of the more popular packages +used to assist in that task is the Mock package. I will no doubt be +labelled as a heretic but when I have tried to use it for things it just +doesn't seem to sit right with my way of thinking. + +It may also just be that what I am trying to apply it to isn't a good fit. +In what I want to test it usually isn't so much that I want to mock out +lower layers, but more that I simply want to validate data being passed +through to the next layer or otherwise modify results. In other words I +usually still need the system as a whole to function end to end and +possibly over an extended time. + +So for the more complex testing I need to do I actually keep falling back +on the monkey patching capabilities of wrapt. It may well just be that +since I wrote wrapt that I am more familiar with its paradigm, or that I +prefer the more explicit way that wrapt requires you to do things. Either +way, for me at least wrapt helps me to get the job done quicker. + +To explain a bit more about the monkey patching capabilities of wrapt, I am +in this blog post going to show how some of the things you can do in Mock +you can do with wrapt. Just keep in mind that I am an absolute novice when +it comes to Mock and so I could also just be too dumb to understand how to +use it properly for what I want to do easily. + +Return values and side effects +------------------------------ + +If one is using Mock and you want to temporarily override the value +returned by a method of a class when called, one way is to use: + +``` +from mock import Mock, patch + +class ProductionClass(object): + def method(self, a, b, c, key): + print a, b, c, key + +@patch(__name__+'.ProductionClass.method', return_value=3) +def test_method(mock_method): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') + mock_method.assert_called_with(3, 4, 5, key='value') + assert result == 3 +``` + +With what I have presented so far of the wrapt package, an equivalent way +of doing this would be: + +``` +from wrapt import patch_function_wrapper + +class ProductionClass(object): + def method(self, a, b, c, key): + print a, b, c, key + +@patch_function_wrapper(__name__, 'ProductionClass.method') +def wrapper(wrapped, instance, args, kwargs): + assert args == (3, 4, 5) and kwargs.get('key') == 'value' + return 3 + +def test_method(): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') + assert result == 3 +``` + +An issue with this though is that the 'wrapt.patch_function_wrapper()' +function I previously described applies a permanent patch. This is okay +where it does need to survive for the life of the process, but in the +case of testing we usually want to only have a patch apply to the single +unit test function being run at that time. So the patch should be +removed at the end of that test and before the next function is called. + +For that scenario, the wrapt package provides an alternate decorator +'@wrapt.transient_function_wrapper'. This can be used to create a wrapper +function that will only be applied for the scope of a specific call that +the decorated function is applied to. We can therefore write the above as: + +``` +from wrapt import transient_function_wrapper + +class ProductionClass(object): + def method(self, a, b, c, key): + print a, b, c, key + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + assert args == (3, 4, 5) and kwargs.get('key') == 'value' + return 3 + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') + assert result == 3 +``` + +Although this example shows how to return a substitute for the method being +called, the more typical case is that I still want to call the original +wrapped function. Thus, perhaps validating the arguments being passed in or +the return value being passed back from the lower layers. + +For this blog post when I tried to work out how to do that with Mock the +general approach I came up with was the following. + +``` +from mock import Mock, patch + +class ProductionClass(object): + def method(self, a, b, c, key): + print a, b, c, key + +def wrapper(wrapped): + def _wrapper(self, *args, **kwargs): + assert args == (3, 4, 5) and kwargs.get('key') == 'value' + return wrapped(self, *args, **kwargs) + return _wrapper + +@patch(__name__+'.ProductionClass.method', autospec=True, + side_effect=wrapper(ProductionClass.method)) + +def test_method(mock_method): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') +``` + +There were two tricks here. The first is the 'autospec=True' argument to +'@Mock.patch' to have it perform method binding, and the second being the +need to capture the original method from the 'ProductionClass' before any +mock had been applied to it, so I could then in turn call it when the side +effect function for the mock was called. + +No doubt someone will tell me that I am doing this all wrong and there is a +simpler way, but that is the best I could come up with after 10 minutes of +reading the Mock documentation. + +When using wrapt to do the same thing, what is used is little different to +what was used when mocking the return value. This is because the wrapt +function wrappers will work with both normal functions or methods and so +nothing special has to be done when wrapping methods. Further, when the +wrapt wrapper function is called, it is always passed the original function +which was wrapped, so no magic is needed to stash that away. + +``` +from wrapt import transient_function_wrapper + +class ProductionClass(object): + def method(self, a, b, c, key): + print a, b, c, key + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + assert args == (3, 4, 5) and kwargs.get('key') == 'value' + return wrapped(*args, **kwargs) + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') +``` + +Using this ability to easily intercept a call to perform validation of data +being passed, but still call the original, I can relatively easily create a +whole bunch of decorators for performing validation on data as is it is +passed through different parts of the system. I can then stack up these +decorators on any test function that I need to add them to. + +Wrapping of return values +------------------------- + +The above recipes cover being able to return a fake return value, returning +the original, or some slight modification of the original where it is some +primitive data type or collection. In some cases though I actually want to +put a wrapper around the return value to modify how subsequent code +interacts with it. + +The first example of this is where the wrapped function returns another +function which would then be called by something higher up the call chain. +Here I may want to put a wrapper around the returned function to allow me +to then intercept when it is called. + +In the case of using Mock I would do something like: + +``` +from mock import Mock, patch + +def function(): + pass + +class ProductionClass(object): + def method(self, a, b, c, key): + return function + +def wrapper2(wrapped): + def _wrapper2(*args, **kwargs): + return wrapped(*args, **kwargs) + return _wrapper2 + +def wrapper1(wrapped): + def _wrapper1(self, *args, **kwargs): + func = wrapped(self, *args, **kwargs) + return Mock(side_effect=wrapper2(func)) + return _wrapper1 + +@patch(__name__+'.ProductionClass.method', autospec=True, + side_effect=wrapper1(ProductionClass.method)) +def test_method(mock_method): + real = ProductionClass() + func = real.method(3, 4, 5, key='value') + result = func() +``` + +And with wrapt I would instead do: + +``` +from wrapt import transient_function_wrapper, function_wrapper + +def function(): + pass + +class ProductionClass(object): + def method(self, a, b, c, key): + return function + +@function_wrapper +def result_function_wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + return result_function_wrapper(wrapped(*args, **kwargs)) + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + func = real.method(3, 4, 5, key='value') + result = func() +``` + +In this example I have used a new decorator called +'@wrapt.function_wrapper'. I could also have used '@wrapt.decorator' in +this example. The '@wrapt.function_wrapper' decorator is actually just a +cut down version of '@wrapt.decorator', lacking some of the bells and +whistles that one doesn't generally need when doing explicit monkey +patching, but otherwise it can be used in the same way. + +I can therefore apply a wrapper around a function returned as a result. I +could could even apply the same principal where a function is being passed +in as an argument to some other function. + +A different scenario to a function being returned is where an instance of a +class is returned. In this case I may want to apply a wrapper around a +specific method of just that instance of the class. + +With the Mock library it again comes down to using its 'Mock' class and +having to apply it in different ways to achieve the result you want. I am +going to step back from Mock now though and just focus on how one can do +things using wrapt. + +So, depending on the requirements there are a couple of ways one could do +this with wrapt. + +The first approach is to replace the method on the instance directly with a +wrapper which encapsulates the original method. + +``` +from wrapt import transient_function_wrapper, function_wrapper + +class StorageClass(object): + def run(self): + pass + +storage = StorageClass() + +class ProductionClass(object): + def method(self, a, b, c, key): + return storage + +@function_wrapper +def run_method_wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + storage = wrapped(*args, **kwargs) + storage.run = run_method_wrapper(storage.run) + return storage + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + data = real.method(3, 4, 5, key='value') + result = data.run() +``` + +This will create the desired result but in this example actually turns out +to be a bad way of doing it. + +The problem in this case is that the object being returned is one which has +a life time beyond the test. That is, we are modifying an object stored at +global scope and which might be used for a different test. By simply +replacing the method on the instance, we have made a permanent change. + +This would be okay if it was a temporary instance of a class created on +demand just for that one call, but not where it is persistent like in this +case. + +We can't therefore modify the instance itself, but need to wrap the +instance in some other way to intercept the method call. + +To do this we make use of what is called an object proxy. This is a special +object type which we can create an instance of to wrap another object. When +accessing the proxy object, any attempts to access attributes will actually +return the attribute from the wrapped object. Similarly, calling a method +on the proxy will call the method on the wrapped object. + +Having a distinct proxy object though allows us to change the behaviour on +the proxy object and so change how code interacts with the wrapped object. +We can therefore avoid needing to change the original object itself. + +For this example what we can therefore do is: + +``` +from wrapt import transient_function_wrapper, ObjectProxy + +class StorageClass(object): + def run(self): + pass + +storage = StorageClass() + +class ProductionClass(object): + def method(self, a, b, c, key): + return storage + +class StorageClassProxy(ObjectProxy): + def run(self): + return self.__wrapped__.run() + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + storage = wrapped(*args, **kwargs) + return StorageClassProxy(storage) + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + data = real.method(3, 4, 5, key='value') + result = data.run() +``` + +That is, we define the 'run()' method on the proxy object to intercept the +call of the same method on the original object. We can then proceed to +return fake values, validate arguments or results, or modify them as +necessary. + +With the proxy we can even intercept access to an attribute of the original +object by adding a property to the proxy object. + +``` +from wrapt import transient_function_wrapper, ObjectProxy + +class StorageClass(object): + def __init__(self): + self.name = 'name' + +storage = StorageClass() + +class ProductionClass(object): + def method(self, a, b, c, key): + return storage + +class StorageClassProxy(ObjectProxy): + @property + def name(self): + return self.__wrapped__.name + +@transient_function_wrapper(__name__, 'ProductionClass.method') +def apply_ProductionClass_method_wrapper(wrapped, instance, args, kwargs): + storage = wrapped(*args, **kwargs) + return StorageClassProxy(storage) + +@apply_ProductionClass_method_wrapper +def test_method(): + real = ProductionClass() + data = real.method(3, 4, 5, key='value') + assert data.name == 'name' +``` + +Building a better Mock +---------------------- + +You might be saying at this point that Mock does a lot more than this. You +might even want to point out how Mock can save away details about the call +which can be checked later at the level of the test harness, rather than +having to resort to raising assertion errors down in the wrappers +themselves which can be an issue if code catches the exceptions before you +see them. + +This is all true, but the goal at this point for wrapt has been to provide +monkey patching mechanisms which do respect introspection, the descriptor +protocol and other things besides. That I can use it for the type of +testing I do is a bonus. + +You aren't limited to using just the basic building blocks themselves +though and personally I think wrapt could be a great base on which to build +a better Mock library for testing. + +I therefore leave you with one final example to get you thinking about the +ways this might be done if you are partial to the way that Mock does +things. + +``` +from wrapt import transient_function_wrapper + +class ProductionClass(object): + def method(self, a, b, c, key): + pass + +def patch(module, name): + def _decorator(wrapped): + class Wrapper(object): + @transient_function_wrapper(module, name) + def __call__(self, wrapped, instance, args, kwargs): + self.args = args + self.kwargs = kwargs + return wrapped(*args, **kwargs) + wrapper = Wrapper() + @wrapper + def _wrapper(): + return wrapped(wrapper) + return _wrapper + return _decorator + +@patch(__name__, 'ProductionClass.method') +def test_method(mock_method): + real = ProductionClass() + result = real.method(3, 4, 5, key='value') + assert real.method.__name__ == 'method' + assert mock_method.args == (3, 4, 5) + assert mock_method.kwargs.get('key') == 'value' +``` + +So that is a quick run down of the main parts of the functionality provided +by wrapt for doing monkey patching. There are a few others things, but that +is in the main all you usually require. I use monkey patching for actually +adding instrumentation into existing code to support performance +monitoring, but I have shown here how the same techniques can be used in +writing tests for your code as an alternative to a package like Mock. + +As I mentioned in my [previous +post](11-safely-applying-monkey-patches-in-python.md) though, one of the +big problems with monkey patching is the order in which modules get +imported relative to when the monkey patching is done. I will talk more +about that issue in the next post. diff --git a/blog/13-ordering-issues-when-monkey-patching-in-python.md b/blog/13-ordering-issues-when-monkey-patching-in-python.md new file mode 100644 index 0000000..5dd0d4c --- /dev/null +++ b/blog/13-ordering-issues-when-monkey-patching-in-python.md @@ -0,0 +1,294 @@ +Ordering issues when monkey patching in Python +============================================== + +In my recent post about [safely applying monkey patches in Python]( +11-safely-applying-monkey-patches-in-python.md), I mentioned how one of the +issues that arises is when a monkey patch is applied. Specifically, if the +module you need to monkey patch has already been imported and was being +used by other code, that it could have created a local reference to a +target function you wish to wrap, in its own namespace. So although your +monkey patch would work fine where the original function was used direct +from the module, you would not cover where it was used via a local +reference. + +Coincidentally, Ned Batchelder recently [posted]( +http://nedbatchelder.com//blog/201503/finding_temp_file_creators.html) +about using monkey patching to debug an issue where temporary directories +were not being cleaned up properly. Ned described this exact issue in +relation to wanting to monkey patch the 'mkdtemp()' function from the +'tempfile' module. In that case he was able to find an alternate place +within the private implementation for the module to patch so as to avoid +the problem. Using some internal function like this may not always be +possible however. + +What I want to start discussing with this post is mechanisms one can use +from wrapt to deal with this issue of ordering. A major part of the +solution is what are called post import hooks. This is a mechanism which +was described in [PEP 369]( +https://www.python.org/dev/peps/pep-0369/) and although it never made it +into the Python core, it is still possible to graft this ability into +Python using existing APIs. From this we can then add additional +capabilities for discovering monkey patching code and automatically apply +it when modules are imported, before other modules get the module and so +before they can create a reference to a function in their own namespace. + +Post import hook mechanism +-------------------------- + +In PEP 369, a primary use case presented was illustrated by the example: + +``` +import imp + +@imp.when_imported('decimal') +def register(decimal): + Inexact.register(decimal.Decimal) +``` + +The basic idea is that when this code was seen it would cause a callback to +be registered within the Python import system such that when the 'decimal' +module was imported, that the 'register()' function which the decorator had +been applied to, would be called. The argument to the 'register()' function +would be the reference to the module the registration had been against. The +function could then perform some action against the module before it was +returned to whatever code originally requested the import. + +Instead of using the decorator '@imp.when_imported' decorator, one could +also explicitly use the 'imp.register_post_import_hook()' function to +register a post import hook. + +``` +import imp + +def register(decimal): + Inexact.register(decimal.Decimal) + +imp.register_post_import_hook(register, 'decimal') +``` + +Although PEP 369 was never incorporated into Python, the wrapt module +provides implementations for both the decorator and the function, but +within the 'wrapt' module rather than 'imp'. + +Now what neither the decorator or the function really solved alone was the +ordering issue. That is, you still had the problem that these could be +triggered after the target module had already been imported. In this case +the post import hook function would still be called, albeit for our case +too late to get in before the reference to the function we want to monkey +patch had been created in a different namespace. + +The simplest solution to this problem is to modify the main Python script +for your application and setup all the post import hook registrations you +need as the absolute very first thing that is done. That is, before any +other modules are imported from your application or even modules from the +standard library used to parse any command line arguments. + +Even if you are able to do this, because though the registration functions +require an actual callable, it does mean you are preloading the code to +perform all the monkey patches. This could be a problem if they in turn had +to import further modules as the state of your application may not yet have +been setup such that those imports would succeed. + +They say though that one level of indirection can solve all problems and +this is an example of where that principle can be applied. That is, rather +than import the monkey patching code, you can setup a registration which +would only lazily load the monkey patching code itself if the module to be +patched was imported, and then execute it. + +``` +import sys + +from wrapt import register_post_import_hook + +def load_and_execute(name): + def _load_and_execute(target_module): + __import__(name) + patch_module = sys.modules[name] + getattr(patch_module, 'apply_patch')(target_module) + return _load_and_execute + +register_post_import_hook(load_and_execute('patch_tempfile'), 'tempfile') +``` + +In the module file 'patch_tempfile.py' we would now have: + +``` +from wrapt import wrap_function_wrapper + +def _mkdtemp_wrapper(wrapped, instance, args, kwargs): + print 'calling', wrapped.__name__ + return wrapped(*args, **kwargs) + +def apply_patch(module): + print 'patching', module.__name__ + wrap_function_wrapper(module, 'mkdtemp', _mkdtemp_wrapper) +``` + +Running the first script with the interactive interpreter so as to leave us +in the interpreter, we can then show what happens when we import the +'tempfile' module and execute the 'mkdtemp()' function. + +``` +$ python -i lazyloader.py +>>> import tempfile +patching tempfile +>>> tempfile.mkdtemp() +calling mkdtemp +'/var/folders/0p/4vcv19pj5d72m_bx0h40sw340000gp/T/tmpfB8r20' +``` + +In other words, unlike how most monkey patching is done, we aren't forcibly +importing a module in order to apply the monkey patches on the basis it +might be used. Instead the monkey patching code stays dormant and unused +until the target module is later imported. If the target module is never +imported, the monkey patch code for that module is itself not even +imported. + +Discovery of post import hooks +------------------------------ + +Post import hooks as described provide a slightly better way of setting up +monkey patches so they are applied. This is because they are only activated +if the target module containing the function to be patched is even +imported. This avoids unnecessarily importing modules you may not even use, +and which otherwise would increase memory usage of your application. + +Ordering is still important and as a result it is important to ensure that +any post import hook registrations are setup before any other modules are +imported. You also need to modify your application code every time you want +to change what monkey patches are applied. This latter point could be +inconvenient if only wanting to add monkey patches infrequently for the +purposes of debugging issues. + +A solution to the latter issue is to separate out monkey patches into +separately installed modules and use a registration mechanism to announce +their availability. Python applications could then have common boiler plate +code executed at the very start which discovers based on supplied +configuration what monkey patches should be applied. The registration +mechanism would then allow the monkey patch modules to be discovered at +runtime. + +One particular registration mechanism which can be used here is +'setuptools' entry points. Using this we can package up monkey patches so +they could be separately installed ready for use. The structure of such a +package would be: + +``` +setup.py +src/__init__.py +src/tempfile_debugging.py +``` + +The 'setup.py' file for this package will be: + +``` +from setuptools import setup + +NAME = 'wrapt_patches.tempfile_debugging' + +def patch_module(module, function=None): + function = function or 'patch_%s' % module.replace('.', '_') + return '%s = %s:%s' % (module, NAME, function) + +ENTRY_POINTS = [ + patch_module('tempfile'), +] + +setup_kwargs = dict( + name = NAME, + version = '0.1', + packages = ['wrapt_patches'], + package_dir = {'wrapt_patches': 'src'}, + entry_points = { NAME: ENTRY_POINTS }, +) + +setup(**setup_kwargs) +``` + +As a convention so that our monkey patch modules are easily identifiable we +use a namespace package. The parent package in this case will be +'wrapt_patches' since we are working with wrapt specifically. + +The name for this specific package will be +'wrapt_patches.tempfile_debugging' as the theoretical intent is that we are +going to create some monkey patches to help us debug use of the 'tempfile' +module, along the lines of what Ned described in his blog post. + +The key part of the 'setup.py' file is the definition of the +'entry_points'. This will be set to a dictionary mapping the package name +to a list of definitions listing what Python modules this package contains +monkey patches for. + +The 'src/__init__.py' file will then contain: + +``` +import pkgutil +__path__ = pkgutil.extend_path(__path__, __name__) +``` + +as is required when creating a namespace package. + +Finally, the monkey patches will actually be contained in +'src/tempfile_debugging.py' and for now is much like what we had before. + +``` +from wrapt import wrap_function_wrapper + +def _mkdtemp_wrapper(wrapped, instance, args, kwargs): + print 'calling', wrapped.__name__ + return wrapped(*args, **kwargs) + +def patch_tempfile(module): + print 'patching', module.__name__ + wrap_function_wrapper(module, 'mkdtemp', _mkdtemp_wrapper) +``` + +With the package defined we would install it into the Python installation +or virtual environment being used. + +In place now of the explicit registrations which we previously added at the +very start of the Python application main script file, we would instead +add: + +``` +import os + +from wrapt import discover_post_import_hooks + +patches = os.environ.get('WRAPT_PATCHES') + +if patches: + for name in patches.split(','): + name = name.strip() + if name: + print 'discover', name + discover_post_import_hooks(name) +``` + +If we were to run the application with no specific configuration to enable +the monkey patches then nothing would happen. If however they were enabled, +then they would be automatically discovered and applied as necessary. + +``` +$ WRAPT_PATCHES=wrapt_patches.tempfile_debugging python -i entrypoints.py +discover wrapt_patches.tempfile_debugging +>>> import tempfile +patching tempfile +``` + +What would be ideal is if PEP 369 ever did make it into the core of Python +that a similar bootstrapping mechanism be incorporated into Python itself +so that it was possible to force registration of monkey patches very early +during interpreter initialisation. Having this in place we would have a +guaranteed way of addressing the ordering issue when doing monkey patching. + +As that doesn't exist right now, what we did in this case was modify our +Python application to add the bootstrap code ourselves. This is fine where +you control the Python application you want to be able to potentially apply +monkey patches to, but what if you wanted to monkey patch a third party +application and you didn't want to have to modify its code. What are the +options in that case? + +As it turns out there are some tricks that can be used in that case. I will +discuss such options for monkey patching a Python application you can't +actually modify in my next blog post on this topic of monkey patching. diff --git a/blog/14-automatic-patching-of-python-applications.md b/blog/14-automatic-patching-of-python-applications.md new file mode 100644 index 0000000..6951c6e --- /dev/null +++ b/blog/14-automatic-patching-of-python-applications.md @@ -0,0 +1,398 @@ +Automatic patching of Python applications +========================================= + +In my [previous posts]( +13-ordering-issues-when-monkey-patching-in-python.md) on monkey patching I +discussed the ordering problem. That is, that the ability to properly +monkey patch is dependent on whether we can get in before any other code +has already imported the module we want to patch. The specific issue in +this case is where other code has imported a reference to a function within +a module by name and stored that in it is own namespace. In other words, +where it has used: + +``` +from module import function +``` + +If we cant get in early enough, then it becomes necessary to monkey patch +all such uses of a target function as well, which in the general case is +impossible as we will not know where the function has been imported. + +Part of the solution I described for this was to use a post import hook +mechanism to allow us to get access to a module for monkey patching before +the module is even returned back to any code where it is being imported. +This technique is still though dependent on the post import hook mechanism +itself being installed before any other code is effectively run. This means +having to manually modify the main Python script file for an application, +something which isnt always practical. + +The point of this post is to look at how we can avoid the need to even +modify that main Python script file. For this there are a few techniques +that could be used. I am going to look at the most evil of those techniques +first and then talk about others in a subsequent post. + +Executable code in .pth files +----------------------------- + +As part of the Python import system and how it determines what directories +are searched for Python modules, there is a mechanism whereby for a package +it is possible to install a file with a .pth extension into the Python +'site-packages' directory. The actual Python package code itself then might +actually be installed in a different location not actually on the Python +module search path, most often actually in a versioned subdirectory of the +'site-packages' directory. The purpose of the .pth file is to act as a +pointer to where the actual code for the Python package lives. + +In the simple case the .pth file will contain a relative or absolute path +name to the name of the actual directory containing the code for the Python +package. In the case of it being a relative path name, then it will be +taken relative to the directory in which the .pth file is located. + +With such .pth files in place, when the Python interpreter is initialising +itself and setting up the Python module search path, after it has added in +all the default directories to be searched, it will look through the +site-packages directory and parse each .pth file, adding to the final list +of directories to be searched any directories specified within the .pth +files. + +Now at one point in the history of Python this .pth mechanism was enhanced +to allow for a special case. This special case was that if a line in the +.pth file started with import, the line would be executed as Python code +instead of simply adding it as a directory to the list of directories to be +searched for modules. + +I am told this originally was to allow special startup code to be executed +for a module to allow registration of a non standard codec for Unicode. It +has though since also been used in the implementation of easy_install and +if you have ever run easy-install and looked at the easy-install.pth file +in the site-packages directory you will find some code which looks like: + +``` +import sys; sys.__plen = len(sys.path) +./antigravity-0.1-py2.7.egg +import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) +``` + +So as long as you can fit the code on one line, you can potentially do some +quite nasty stuff inside of a .pth file every time that the Python +interpreter is run. + +Personally I find the concept of executable code inside of a .pth file +really dangerous and up until now have avoided relying on this feature of +.pth files. + +My concerns over executable code in .pth files is the fact that it is +always run. This means that even if you had installed a pre built RPM/DEB +package or a Python wheel into a system wide Python installation, with the +idea that this was somehow much safer because you were avoiding running the +setup.py file for a package as the root user, the .pth file means that the +package can still subsequently run code without you realising and without +you even having imported the module into any application. + +If one wanted to be paranoid about security, then Python should really have +a whitelisting mechanism for what .pth files you wanted to trust and allow +code to be executed from every time the Python interpreter is run, +especially as the root user. + +I will leave that discussion up to others if anyone else cares to be +concerned and for now at least will show how this feature of .pth files can +be used (abused) to implement a mechanism for automated monkey patching of +any Python application being run. + +Adding Python import hooks +-------------------------- + +In the previous post where I talked about the post import hook mechanism, +the code I gave as needing to be able to be manually added at the start of +any Python application script file was: + +``` +import os + +from wrapt import discover_post_import_hooks + +patches = os.environ.get('WRAPT_PATCHES') + +if patches: + for name in patches.split(','): + name = name.strip() + if name: + print 'discover', name + discover_post_import_hooks(name) +``` + +What this was doing was using an environment variable as the source of +names for any packages registered using setuptools entry points that +contained monkey patches we wanted to have applied. + +Knowing about the ability to have executable code in .pth files, lets now +work out how we can use that to instead have this code executed +automatically every time the Python interpreter is run, thereby avoiding +the need to manually modify every Python application we want to have monkey +patches applied to. + +In practice however, the code we will need is actually going to have to be +slightly more complicated than this and as a result not something that can +be readily added directly to a .pth file due to the limitation of code +needing to all be on one line. What we will therefore do is put all our +code in a separate module and execute it from there. We dont want to be too +nasty and import that module every time though, perhaps scaring users when +they see it imported even if not used, so we will gate even that by the +presence of the environment variable. + +What we can therefore use in our ‘.pth’ is: + +``` +import os, sys; os.environ.get('AUTOWRAPT_BOOTSTRAP') and __import__('autowrapt.bootstrap') and sys.modules['autowrapt.bootstrap'].bootstrap() +``` + +That is, if the environment variable is set to a non empty value only then +do we import our module containing our bootstrap code and execute it. + +As to the bootstrap code, this is where things get a bit messy. We cant +just use the code we had used before when manually modifying the Python +application script file. This is because of where in the Python interpreter +initialisation the parsing of .pth files is done. + +The problems are twofold. The first issue with executing the discovery of +the import hooks directly when the .pth file is processed is that the order +in which they are processed is unknown and so at the point our code is run +the final Python module search path may not have been setup. The second +issue is that .pth file processing is done before any sitecustomize.py or +usercustomize.py processing has been done. The Python interpreter therefore +may not be in its final configured state. We therefore have to be a little +bit careful of what we do. + +What we really want is to defer any actions until the Python interpreter +initialisation has been completed. The problem is how we achieve that. + +Python interpreter ‘site’ module +-------------------------------- + +The actual final parts of Python interpreter initialisation is performed +from the main() function of the site module: + +``` +def main(): + global ENABLE_USER_SITE + abs__file__() + known_paths = removeduppaths() + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + known_paths = addusersitepackages(known_paths) + known_paths = addsitepackages(known_paths) + if sys.platform == 'os2emx': + setBEGINLIBPATH() + setquit() + setcopyright() + sethelper() + aliasmbcs() + setencoding() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + # Remove sys.setdefaultencoding() so that users cannot change the + # encoding after initialization. The test for presence is needed when + # this module is run as a script, because this code is executed twice. + if hasattr(sys, "setdefaultencoding"): + del sys.setdefaultencoding +``` + +The .pth parsing and code execution we want to rely upon is done within the +addsitepackages() function. + +What we really want therefore is to defer any execution of our code until +after the functions execsitecustomize() or execusercustomize() are run. The +way to achieve that is to monkey patch those two functions and trigger our +code when they have completed. + +We have to monkey patch both because the usercustomize.py processing is +optional dependent on whether ENABLE_USER_SITE is true or not. Our +'bootstrap() function therefore needs to look like: + +``` +def _execsitecustomize_wrapper(wrapped): + def _execsitecustomize(*args, **kwargs): + try: + return wrapped(*args, **kwargs) + finally: + if not site.ENABLE_USER_SITE: + _register_bootstrap_functions() + return _execsitecustomize + +def _execusercustomize_wrapper(wrapped): + def _execusercustomize(*args, **kwargs): + try: + return wrapped(*args, **kwargs) + finally: + _register_bootstrap_functions() + return _execusercustomize + +def bootstrap(): + site.execsitecustomize = _execsitecustomize_wrapper(site.execsitecustomize) + site.execusercustomize = _execusercustomize_wrapper(site.execusercustomize) +``` + +Despite everything I have ever said about how manually constructed monkey +patches is bad and that the wrapt module should be used for doing monkey +patching, we cant actually use the wrapt module in this case. This is +because technically, as a user installed package, the wrapt package may not +be usable at this point. This could occur where wrapt was installed in such +a way that the ability to import it was itself dependent on the processing +of .pth files. As a result we drop down to using a simple wrapper using a +function closure. + +In the actual wrappers, you can see how which of the two wrappers actually +ends up calling _register_bootstrap_functions() is dependent on whether +ENABLE_USER_SITE is true or not, only calling it in execsitecustomize() if +support for usersitecustomize was enabled. + +Finally we now have our '_register_bootstrap_functions()’ defined as: + +``` +_registered = False + +def _register_bootstrap_functions(): + global _registered + if _registered: + return + _registered = True + + from wrapt import discover_post_import_hooks + for name in os.environ.get('AUTOWRAPT_BOOTSTRAP', '').split(','): + discover_post_import_hooks(name) +``` + +Bundling it up as a package +--------------------------- + +We have worked out the various bits we require, but how do we get this +installed, in particular how do we get the custom .pth file installed. For +that we use a setup.py file of: + +``` +import sys +import os + +from setuptools import setup +from distutils.sysconfig import get_python_lib + +setup_kwargs = dict( + name = 'autowrapt', + packages = ['autowrapt'], + package_dir = {'autowrapt': 'src'}, + data_files = [(get_python_lib(prefix=''), ['autowrapt-init.pth'])], + entry_points = {'autowrapt.examples’: ['this = autowrapt.examples:autowrapt_this']}, + install_requires = ['wrapt>=1.10.4'], +) + +setup(**setup_kwargs) +``` + +To get that .pth installed we have used the data_files argument to the +setup() call. The actual location for installing the file is determined +using the get_python_lib() function from the distutils.sysconfig module. +The prefix' argument of an empty string ensures that a relative path for +the site-packages directory where Python packages should be installed is +used rather than an absolute path. + +Very important when installing this package though is that you cannot use +easy_install or python setup.py install. One can only install this package +using pip. + +The reason for this is that if not using pip, then the package installation +tool can install the package as an egg. In this situation the custom .pth +file will actually be installed within the egg directory and not actually +within the site-packages directory. + +The only .pth file added to the site-packages directory will be that used +to map that the autowrapt package exists in the sub directory. The +addsitepackages() function called from the site module doesnt in turn +process .pth files contained in a directory added by a .pth file, so our +custom .pth file would be skipped. + +When using ‘pip’ it doesn’t use eggs by default and so we are okay. + +Also do be aware that this package will not work with buildout as it will +always install packages as eggs and explicitly sets up the Python module +search path itself in any Python scripts installed into the Python +installation. + +Trying out an example +--------------------- + +The actual complete source code for this package can be found at: + +* https://github.com/GrahamDumpleton/autowrapt + +The package has also been released on PyPi as autowrapt so you can actually +try it, and use it if you really want to. + +To allow for a easy quick test to see that it works, the autowrapt package +bundles an example monkey patch. In the above setup.py this was set up by: + +``` +entry_points = {'autowrapt.examples’: ['this = autowrapt.examples:autowrapt_this']}, +``` + +This entry point definition names a monkey patch with the name +autowrapt.examples. The definition says that when the this module is +installed, the monkey patch function autowrapt_this() in the module +autowrapt.examples will be called. + +So to run the test do: + +``` +pip install autowrapt +``` + +This should also install the wrapt module if you dont have the required +minimum version. + +Now run the command line interpreter as normal and at the prompt do: + +``` +import this +``` + +This should result in the Zen of Python being displayed. + +Exit the Python interpreter and now instead run: + +``` +AUTOWRAPT_BOOTSTRAP=autowrapt.examples python +``` + +This runs the Python interpreter again, but also sets the environment +variable AUTOWRAPT_BOOTSTRAP with the value autowrapt.examples matching the +name of the entry point defined in the setup.py file for autowrapt'. + +The actual code for the ‘autowrapt_this()’ function was: + +``` +from __future__ import print_function + +def autowrapt_this(module): + print('The wrapt package is absolutely amazing and you should use it.') +``` + +so if we now again run: + +``` +import this +``` + +we should now see an extended version of the Zen of Python. + +We didnt actually monkey patch any code in the target module in this case, +but it shows that the monkey patch function was actually triggered when +expected. + +Other bootstrapping mechanisms +------------------------------ + +Although this mechanism is reasonably clean and only requires the setting +of an environment variable, it cannot be used with buildout as mentioned. +For buildout we need to investigate other approaches we could use to +achieve the same affect. I will cover such other options in the next blog +post on this topic. diff --git a/blog/README.md b/blog/README.md index 334306c..2497940 100644 --- a/blog/README.md +++ b/blog/README.md @@ -1,6 +1,9 @@ Blog posts ========== +Decorators (2014) +----------------- + * 01 - [How you implemented your Python decorator is wrong](01-how-you-implemented-your-python-decorator-is-wrong.md) - (7th January 2014) * 02 - [The interaction between decorators and descriptors](02-the-interaction-between-decorators-and-descriptors.md) - (7th January 2014) * 03 - [Implementing a factory for creating decorators](03-implementing-a-factory-for-creating-decorators.md) - (8th January 2014) @@ -11,3 +14,11 @@ Blog posts * 08 - [The @synchronized decorator as context manager](08-the-synchronized-decorator-as-context-manager.md) - (15th January 2014) * 09 - [Performance overhead of using decorators](09-performance-overhead-of-using-decorators.md) - (8th February 2014) * 10 - [Performance overhead when applying decorators to methods](10-performance-overhead-when-applying-decorators-to-methods.md) - (17th February 2014) + +Monkey Patching (2015) +---------------------- + +* 11 - [Safely applying monkey patches in Python](11-safely-applying-monkey-patches-in-python.md) - (11th March 2015) +* 12 - [Using wrapt to support testing of software](12-using-wrapt-to-support-testing-of-software.md) - (12th March 2015) +* 13 - [Ordering issues when monkey patching in Python](13-ordering-issues-when-monkey-patching-in-python.md) - (18th March 2015) +* 14 - [Automatic patching of Python applications](14-automatic-patching-of-python-applications.md) - (9th April 2014) From 5461cd070d1e31c1f5b4b60f42a109560226be49 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 14 Apr 2015 11:12:49 -0400 Subject: [PATCH 6/8] Fix formatting of examples in blog post. --- blog/11-safely-applying-monkey-patches-in-python.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/blog/11-safely-applying-monkey-patches-in-python.md b/blog/11-safely-applying-monkey-patches-in-python.md index 2924860..8ea1435 100644 --- a/blog/11-safely-applying-monkey-patches-in-python.md +++ b/blog/11-safely-applying-monkey-patches-in-python.md @@ -55,6 +55,7 @@ primary pattern for this was: ``` import wrapt import inspect + @wrapt.decorator def universal(wrapped, instance, args, kwargs): if instance is None: @@ -138,6 +139,7 @@ In effect you are doing: class Example(object): def name(self): return 'name' + Example.name = universal(Example.name) ``` @@ -155,6 +157,7 @@ class Example(object): def name(self): return 'name' print type(name) + print type(Example.name) ``` @@ -195,6 +198,7 @@ To add the wrapper function to a target function we now use the class Example(object): def name(self): return 'name' + import wrapt wrapt.wrap_function_wrapper(Example, 'name', wrapper) @@ -207,6 +211,7 @@ done: import example import wrapt + wrapt.wrap_function_wrapper(example, 'Example.name', wrapper) ``` @@ -219,6 +224,7 @@ of the module. ``` import wrapt + wrapt.wrap_function_wrapper('example', 'Example.name', wrapper) ``` @@ -262,6 +268,7 @@ If you come along later and have: ``` import wrapt + @wrapt.patch_function_wrapper('example', 'function') def wrapper(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) From b9feaf5e4347b8a4bc1828cf5192433e74ec6cc4 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 21 Jun 2015 22:21:03 +1000 Subject: [PATCH 7/8] Add weakref support for C implementation of function wrapper. --- docs/changes.rst | 4 ++++ src/_wrappers.c | 15 ++++++++++----- src/wrappers.py | 14 ++++++++++++++ tests/test_weak_function_proxy.py | 18 ++++++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index e6bf0d2..3273d21 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -10,6 +10,10 @@ Version 1.10.5 target modules were registered in the same entry point list. Only the callback for the last would be called regardless of the target module. +* If a ``WeakFunctionProxy`` wrapper was used around a method of a class + which was decorated using a wrapt decorator, the decorator wasn't being + invoked when the method was called via the weakref proxy. + **Features Changed** * The ``register_post_import_hook()`` function, modelled after the diff --git a/src/_wrappers.c b/src/_wrappers.c index e28e1bf..9194f54 100644 --- a/src/_wrappers.c +++ b/src/_wrappers.c @@ -15,6 +15,7 @@ typedef struct { PyObject *dict; PyObject *wrapped; + PyObject *weakreflist; } WraptObjectProxyObject; PyTypeObject WraptObjectProxy_Type; @@ -48,6 +49,7 @@ static PyObject *WraptObjectProxy_new(PyTypeObject *type, self->dict = PyDict_New(); self->wrapped = NULL; + self->weakreflist = NULL; return (PyObject *)self; } @@ -153,6 +155,9 @@ static void WraptObjectProxy_dealloc(WraptObjectProxyObject *self) { PyObject_GC_UnTrack(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *)self); + WraptObjectProxy_clear(self); Py_TYPE(self)->tp_free(self); @@ -1683,7 +1688,7 @@ PyTypeObject WraptObjectProxy_Type = { (traverseproc)WraptObjectProxy_traverse, /*tp_traverse*/ (inquiry)WraptObjectProxy_clear, /*tp_clear*/ (richcmpfunc)WraptObjectProxy_richcompare, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ + offsetof(WraptObjectProxyObject, weakreflist), /*tp_weaklistoffset*/ (getiterfunc)WraptObjectProxy_iter, /*tp_iter*/ 0, /*tp_iternext*/ WraptObjectProxy_methods, /*tp_methods*/ @@ -1754,7 +1759,7 @@ PyTypeObject WraptCallableObjectProxy_Type = { 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ + offsetof(WraptObjectProxyObject, weakreflist), /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ 0, /*tp_methods*/ @@ -2249,7 +2254,7 @@ PyTypeObject WraptFunctionWrapperBase_Type = { (traverseproc)WraptFunctionWrapperBase_traverse, /*tp_traverse*/ (inquiry)WraptFunctionWrapperBase_clear, /*tp_clear*/ 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ + offsetof(WraptObjectProxyObject, weakreflist), /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ 0, /*tp_methods*/ @@ -2482,7 +2487,7 @@ PyTypeObject WraptBoundFunctionWrapper_Type = { 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ + offsetof(WraptObjectProxyObject, weakreflist), /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ 0, /*tp_methods*/ @@ -2622,7 +2627,7 @@ PyTypeObject WraptFunctionWrapper_Type = { 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ + offsetof(WraptObjectProxyObject, weakreflist), /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ 0, /*tp_methods*/ diff --git a/src/wrappers.py b/src/wrappers.py index dbc8a0f..945caaf 100644 --- a/src/wrappers.py +++ b/src/wrappers.py @@ -855,6 +855,20 @@ class WeakFunctionProxy(ObjectProxy): self._self_expired = False + if isinstance(wrapped, _FunctionWrapperBase): + self._self_instance = weakref.ref(wrapped._self_instance, + _callback) + + if wrapped._self_parent is not None: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped._self_parent, _callback)) + + else: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped, _callback)) + + return + try: self._self_instance = weakref.ref(wrapped.__self__, _callback) diff --git a/tests/test_weak_function_proxy.py b/tests/test_weak_function_proxy.py index 0f5bd09..d34da0e 100644 --- a/tests/test_weak_function_proxy.py +++ b/tests/test_weak_function_proxy.py @@ -172,5 +172,23 @@ class TestWeakFunctionProxy(unittest.TestCase): self.assertEqual(len(result), 1) self.assertEqual(id(proxy), result[0]) + def test_decorator_method(self): + @wrapt.decorator + def bark(wrapped, instance, args, kwargs): + return 'bark' + + class Animal(object): + @bark + def squeal(self): + return 'squeal' + + animal = Animal() + + self.assertEqual(animal.squeal(), 'bark') + + method = wrapt.WeakFunctionProxy(animal.squeal) + + self.assertEqual(method(), 'bark') + if __name__ == '__main__': unittest.main() From 310d7a41239b51bc4ec521d3303452e2a4cfb68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Starck?= Date: Thu, 25 Jun 2015 14:04:47 -0400 Subject: [PATCH 8/8] Removed (unnecessary) double assignment in ObjectProxy.__setattr__ I would be rather surprised if this was desired, and if yes I'd ask : why is this necessary ? --- src/wrappers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wrappers.py b/src/wrappers.py index 945caaf..7e183f6 100644 --- a/src/wrappers.py +++ b/src/wrappers.py @@ -170,7 +170,6 @@ class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): object.__delattr__(self, '__qualname__') except AttributeError: pass - object.__setattr__(self, name, value) try: object.__setattr__(self, '__qualname__', value.__qualname__) except AttributeError: