diff --git a/newrelic/common/object_wrapper.py b/newrelic/common/object_wrapper.py index 7d9824fe0c..c676966108 100644 --- a/newrelic/common/object_wrapper.py +++ b/newrelic/common/object_wrapper.py @@ -19,16 +19,19 @@ """ -import sys import inspect -from newrelic.packages import six - -from newrelic.packages.wrapt import (ObjectProxy as _ObjectProxy, - FunctionWrapper as _FunctionWrapper, - BoundFunctionWrapper as _BoundFunctionWrapper) - -from newrelic.packages.wrapt.wrappers import _FunctionWrapperBase +from newrelic.packages.wrapt import BoundFunctionWrapper as _BoundFunctionWrapper +from newrelic.packages.wrapt import CallableObjectProxy as _CallableObjectProxy +from newrelic.packages.wrapt import FunctionWrapper as _FunctionWrapper +from newrelic.packages.wrapt import ObjectProxy as _ObjectProxy +from newrelic.packages.wrapt import ( # noqa: F401; pylint: disable=W0611 + apply_patch, + resolve_path, + wrap_object, + wrap_object_attribute, +) +from newrelic.packages.wrapt.__wrapt__ import _FunctionWrapperBase # We previously had our own pure Python implementation of the generic # object wrapper but we now defer to using the wrapt module as its C @@ -47,28 +50,36 @@ # ObjectProxy or FunctionWrapper should be used going forward. -class _ObjectWrapperBase(object): +class ObjectProxy(_ObjectProxy): + """ + This class provides method overrides for all object wrappers used by the + agent. These methods allow attributes to be defined with the special prefix + _nr_ to be interpretted as attributes on the wrapper, rather than the + wrapped object. Inheriting from the base class wrapt.ObjectProxy preserves + method resolution order (MRO) through multiple inheritance. + (See https://www.python.org/download/releases/2.3/mro/). + """ def __setattr__(self, name, value): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) setattr(self, name, value) else: - _ObjectProxy.__setattr__(self, name, value) + super(ObjectProxy, self).__setattr__(name, value) def __getattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) return getattr(self, name) else: - return _ObjectProxy.__getattr__(self, name) + return super(ObjectProxy, self).__getattr__(name) def __delattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) delattr(self, name) else: - _ObjectProxy.__delattr__(self, name) + super(ObjectProxy, self).__delattr__(name) @property def _nr_next_object(self): @@ -79,8 +90,7 @@ def _nr_last_object(self): try: return self._self_last_object except AttributeError: - self._self_last_object = getattr(self.__wrapped__, - '_nr_last_object', self.__wrapped__) + self._self_last_object = getattr(self.__wrapped__, "_nr_last_object", self.__wrapped__) return self._self_last_object @property @@ -96,166 +106,45 @@ def _nr_parent(self): return self._self_parent -class _NRBoundFunctionWrapper(_ObjectWrapperBase, _BoundFunctionWrapper): +class _NRBoundFunctionWrapper(ObjectProxy, _BoundFunctionWrapper): pass -class FunctionWrapper(_ObjectWrapperBase, _FunctionWrapper): +class FunctionWrapper(ObjectProxy, _FunctionWrapper): __bound_function_wrapper__ = _NRBoundFunctionWrapper -class ObjectProxy(_ObjectProxy): - - def __setattr__(self, name, value): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - setattr(self, name, value) - else: - _ObjectProxy.__setattr__(self, name, value) - - def __getattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - return getattr(self, name) - else: - return _ObjectProxy.__getattr__(self, name) - - def __delattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - delattr(self, name) - else: - _ObjectProxy.__delattr__(self, name) - - @property - def _nr_next_object(self): - return self.__wrapped__ - - @property - def _nr_last_object(self): - try: - return self._self_last_object - except AttributeError: - self._self_last_object = getattr(self.__wrapped__, - '_nr_last_object', self.__wrapped__) - return self._self_last_object - - -class CallableObjectProxy(ObjectProxy): +class CallableObjectProxy(ObjectProxy, _CallableObjectProxy): + pass - def __call__(self, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) # The ObjectWrapper class needs to be deprecated and removed once all our # own code no longer uses it. It reaches down into what are wrapt internals # at present which shouldn't be doing. -class ObjectWrapper(_ObjectWrapperBase, _FunctionWrapperBase): +class ObjectWrapper(ObjectProxy, _FunctionWrapperBase): __bound_function_wrapper__ = _NRBoundFunctionWrapper def __init__(self, wrapped, instance, wrapper): if isinstance(wrapped, classmethod): - binding = 'classmethod' + binding = "classmethod" elif isinstance(wrapped, staticmethod): - binding = 'staticmethod' + binding = "staticmethod" else: - binding = 'function' - - super(ObjectWrapper, self).__init__(wrapped, instance, wrapper, - binding=binding) - - -# Helper functions for performing monkey patching. + binding = "function" + super(ObjectWrapper, self).__init__(wrapped, instance, wrapper, binding=binding) -def resolve_path(module, name): - if isinstance(module, six.string_types): - __import__(module) - module = sys.modules[module] - - parent = module - - path = name.split('.') - attribute = path[0] - - original = getattr(parent, attribute) - for attribute in path[1:]: - parent = original - - # We can't just always use getattr() because in doing - # that on a class it will cause binding to occur which - # will complicate things later and cause some things not - # to work. For the case of a class we therefore access - # the __dict__ directly. To cope though with the wrong - # class being given to us, or a method being moved into - # a base class, we need to walk the class hierarchy to - # work out exactly which __dict__ the method was defined - # in, as accessing it from __dict__ will fail if it was - # not actually on the class given. Fallback to using - # getattr() if we can't find it. If it truly doesn't - # exist, then that will fail. - - if inspect.isclass(original): - for cls in inspect.getmro(original): - if attribute in vars(cls): - original = vars(cls)[attribute] - break - else: - original = getattr(original, attribute) - - else: - original = getattr(original, attribute) - - return (parent, attribute, original) - - -def apply_patch(parent, attribute, replacement): - setattr(parent, attribute, replacement) - - -def wrap_object(module, name, factory, args=(), kwargs={}): - (parent, attribute, original) = resolve_path(module, name) - wrapper = factory(original, *args, **kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Function for apply a proxy object to an attribute of a class instance. -# The wrapper works by defining an attribute of the same name on the -# class which is a descriptor and which intercepts access to the -# instance attribute. Note that this cannot be used on attributes which -# are themselves defined by a property object. - - -class AttributeWrapper(object): - - def __init__(self, attribute, factory, args, kwargs): - self.attribute = attribute - self.factory = factory - self.args = args - self.kwargs = kwargs - - def __get__(self, instance, owner): - value = instance.__dict__[self.attribute] - return self.factory(value, *self.args, **self.kwargs) - - def __set__(self, instance, value): - instance.__dict__[self.attribute] = value - - def __delete__(self, instance): - del instance.__dict__[self.attribute] - - -def wrap_object_attribute(module, name, factory, args=(), kwargs={}): - path, attribute = name.rsplit('.', 1) - parent = resolve_path(module, path)[2] - wrapper = AttributeWrapper(attribute, factory, args, kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper # Function for creating a decorator for applying to functions, as well as # short cut functions for applying wrapper functions via monkey patching. +# WARNING: These functions are reproduced directly from wrapt, but using +# our FunctionWrapper class which includes the _nr_ attriubte overrides +# that are inherited from our subclass of wrapt.ObjectProxy.These MUST be +# kept in sync with wrapt when upgrading, or drift may introduce bugs. + def function_wrapper(wrapper): def _wrapper(wrapped, instance, args, kwargs): @@ -267,6 +156,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: target_wrapper = wrapper.__get__(instance, type(instance)) return FunctionWrapper(target_wrapped, target_wrapper) + return FunctionWrapper(wrapper, _wrapper) @@ -274,9 +164,10 @@ def wrap_function_wrapper(module, name, wrapper): return wrap_object(module, name, FunctionWrapper, (wrapper,)) -def patch_function_wrapper(module, name): +def patch_function_wrapper(module, name, enabled=None): def _wrapper(wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) + return wrap_object(module, name, FunctionWrapper, (wrapper, enabled)) + return _wrapper @@ -299,10 +190,14 @@ def _execute(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) finally: setattr(parent, attribute, original) + return FunctionWrapper(target_wrapped, _execute) + return FunctionWrapper(wrapper, _wrapper) + return _decorator + # Generic decorators for performing actions before and after a wrapped # function is called, or modifying the inbound arguments or return value. @@ -315,6 +210,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: function(*args, **kwargs) return wrapped(*args, **kwargs) + return _wrapper @@ -335,6 +231,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: function(*args, **kwargs) return result + return _wrapper @@ -382,6 +279,7 @@ def out_function(function): @function_wrapper def _wrapper(wrapped, instance, args, kwargs): return function(wrapped(*args, **kwargs)) + return _wrapper diff --git a/newrelic/packages/wrapt/__init__.py b/newrelic/packages/wrapt/__init__.py index ee6539b774..ed31a94313 100644 --- a/newrelic/packages/wrapt/__init__.py +++ b/newrelic/packages/wrapt/__init__.py @@ -1,12 +1,15 @@ -__version_info__ = ('1', '14', '1') +__version_info__ = ('1', '16', '0') __version__ = '.'.join(__version_info__) -from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper, - BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy, - resolve_path, apply_patch, wrap_object, wrap_object_attribute, +from .__wrapt__ import (ObjectProxy, CallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, PartialCallableObjectProxy) + +from .patches import (resolve_path, apply_patch, wrap_object, wrap_object_attribute, function_wrapper, wrap_function_wrapper, patch_function_wrapper, transient_function_wrapper) +from .weakrefs import WeakFunctionProxy + from .decorators import (adapter_factory, AdapterFactory, decorator, synchronized) diff --git a/newrelic/packages/wrapt/__wrapt__.py b/newrelic/packages/wrapt/__wrapt__.py new file mode 100644 index 0000000000..9933b2c972 --- /dev/null +++ b/newrelic/packages/wrapt/__wrapt__.py @@ -0,0 +1,14 @@ +import os + +from .wrappers import (ObjectProxy, CallableObjectProxy, + PartialCallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, _FunctionWrapperBase) + +try: + if not os.environ.get('WRAPT_DISABLE_EXTENSIONS'): + from ._wrappers import (ObjectProxy, CallableObjectProxy, + PartialCallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, _FunctionWrapperBase) + +except ImportError: + pass diff --git a/newrelic/packages/wrapt/_wrappers.c b/newrelic/packages/wrapt/_wrappers.c index 67c5d5e1af..e0e1b5bc65 100644 --- a/newrelic/packages/wrapt/_wrappers.c +++ b/newrelic/packages/wrapt/_wrappers.c @@ -1139,6 +1139,30 @@ static int WraptObjectProxy_setitem(WraptObjectProxyObject *self, /* ------------------------------------------------------------------------- */ +static PyObject *WraptObjectProxy_self_setattr( + WraptObjectProxyObject *self, PyObject *args) +{ + PyObject *name = NULL; + PyObject *value = NULL; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "UO:__self_setattr__", &name, &value)) + return NULL; +#else + if (!PyArg_ParseTuple(args, "SO:__self_setattr__", &name, &value)) + return NULL; +#endif + + if (PyObject_GenericSetAttr((PyObject *)self, name, value) != 0) { + return NULL; + } + + Py_INCREF(Py_None); + return Py_None; +} + +/* ------------------------------------------------------------------------- */ + static PyObject *WraptObjectProxy_dir( WraptObjectProxyObject *self, PyObject *args) { @@ -1464,6 +1488,19 @@ static PyObject *WraptObjectProxy_get_class( /* ------------------------------------------------------------------------- */ +static int WraptObjectProxy_set_class(WraptObjectProxyObject *self, + PyObject *value) +{ + if (!self->wrapped) { + PyErr_SetString(PyExc_ValueError, "wrapper has not been initialized"); + return -1; + } + + return PyObject_SetAttrString(self->wrapped, "__class__", value); +} + +/* ------------------------------------------------------------------------- */ + static PyObject *WraptObjectProxy_get_annotations( WraptObjectProxyObject *self) { @@ -1535,6 +1572,9 @@ static PyObject *WraptObjectProxy_getattro( if (object) return object; + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + return NULL; + PyErr_Clear(); if (!getattr_str) { @@ -1738,6 +1778,8 @@ static PyMappingMethods WraptObjectProxy_as_mapping = { }; static PyMethodDef WraptObjectProxy_methods[] = { + { "__self_setattr__", (PyCFunction)WraptObjectProxy_self_setattr, + METH_VARARGS , 0 }, { "__dir__", (PyCFunction)WraptObjectProxy_dir, METH_NOARGS, 0 }, { "__enter__", (PyCFunction)WraptObjectProxy_enter, METH_VARARGS | METH_KEYWORDS, 0 }, @@ -1776,7 +1818,7 @@ static PyGetSetDef WraptObjectProxy_getset[] = { { "__doc__", (getter)WraptObjectProxy_get_doc, (setter)WraptObjectProxy_set_doc, 0 }, { "__class__", (getter)WraptObjectProxy_get_class, - NULL, 0 }, + (setter)WraptObjectProxy_set_class, 0 }, { "__annotations__", (getter)WraptObjectProxy_get_annotations, (setter)WraptObjectProxy_set_annotations, 0 }, { "__wrapped__", (getter)WraptObjectProxy_get_wrapped, @@ -2547,7 +2589,6 @@ static PyObject *WraptFunctionWrapperBase_set_name( static PyObject *WraptFunctionWrapperBase_instancecheck( WraptFunctionWrapperObject *self, PyObject *instance) { - PyObject *object = NULL; PyObject *result = NULL; int check = 0; diff --git a/newrelic/packages/wrapt/decorators.py b/newrelic/packages/wrapt/decorators.py index c3f2547295..c80a4bb72e 100644 --- a/newrelic/packages/wrapt/decorators.py +++ b/newrelic/packages/wrapt/decorators.py @@ -41,7 +41,7 @@ def exec_(_code_, _globs_=None, _locs_=None): except ImportError: pass -from .wrappers import (FunctionWrapper, BoundFunctionWrapper, ObjectProxy, +from .__wrapt__ import (FunctionWrapper, BoundFunctionWrapper, ObjectProxy, CallableObjectProxy) # Adapter wrapper for the wrapped function which will overlay certain diff --git a/newrelic/packages/wrapt/importer.py b/newrelic/packages/wrapt/importer.py index 02686e491e..23fcbd2f63 100644 --- a/newrelic/packages/wrapt/importer.py +++ b/newrelic/packages/wrapt/importer.py @@ -9,13 +9,13 @@ PY2 = sys.version_info[0] == 2 if PY2: - string_types = (basestring,) + string_types = basestring, find_spec = None else: - string_types = (str,) + string_types = str, from importlib.util import find_spec -from .decorators import synchronized +from .__wrapt__ import ObjectProxy # The dictionary registering any post import hooks to be triggered once # the target module has been imported. Once a module has been imported @@ -34,21 +34,17 @@ # proxy callback being registered which will defer loading of the # specified module containing the callback function until required. - def _create_import_hook_from_string(name): def import_hook(module): - module_name, function = name.split(":") - attrs = function.split(".") + module_name, function = name.split(':') + attrs = function.split('.') __import__(module_name) callback = sys.modules[module_name] for attr in attrs: callback = getattr(callback, attr) return callback(module) - return import_hook - -@synchronized(_post_import_hooks_lock) def register_post_import_hook(hook, name): # Create a deferred import hook if hook is a string name rather than # a callable function. @@ -56,55 +52,34 @@ def register_post_import_hook(hook, name): if isinstance(hook, string_types): hook = _create_import_hook_from_string(hook) - # Automatically install the import hook finder if it has not already - # been installed. - - global _post_import_hooks_init + with _post_import_hooks_lock: + # Automatically install the import hook finder if it has not already + # been installed. - if not _post_import_hooks_init: - _post_import_hooks_init = True - sys.meta_path.insert(0, ImportHookFinder()) + global _post_import_hooks_init - # Determine if any prior registration of a post import hook for - # the target modules has occurred and act appropriately. + if not _post_import_hooks_init: + _post_import_hooks_init = True + sys.meta_path.insert(0, ImportHookFinder()) - hooks = _post_import_hooks.get(name, None) - - if hooks is None: - # No prior registration of post import hooks for the target - # module. We need to check whether the module has already been - # imported. If it has we fire the hook immediately and add an - # empty list to the registry to indicate that the module has - # already been imported and hooks have fired. Otherwise add - # the post import hook to the registry. + # Check if the module is already imported. If not, register the hook + # to be called after import. module = sys.modules.get(name, None) - if module is not None: - _post_import_hooks[name] = [] - hook(module) - - else: - _post_import_hooks[name] = [hook] + if module is None: + _post_import_hooks.setdefault(name, []).append(hook) - elif hooks == []: - # A prior registration of port import hooks for the target - # module was done and the hooks already fired. Fire the hook - # immediately. + # If the module is already imported, we fire the hook right away. Note that + # the hook is called outside of the lock to avoid deadlocks if code run as a + # consequence of calling the module import hook in turn triggers a separate + # thread which tries to register an import hook. - module = sys.modules[name] + if module is not None: hook(module) - else: - # A prior registration of port import hooks for the target - # module was done but the module has not yet been imported. - - _post_import_hooks[name].append(hook) - - # Register post import hooks defined as package entry points. - def _create_import_hook_from_entrypoint(entrypoint): def import_hook(module): __import__(entrypoint.module_name) @@ -112,74 +87,88 @@ def import_hook(module): for attr in entrypoint.attrs: callback = getattr(callback, attr) return callback(module) - return import_hook - def discover_post_import_hooks(group): try: - # Preferred after Python 3.10 - if sys.version_info >= (3, 10): - from importlib.metadata import entry_points - # Introduced in Python 3.8 - elif sys.version_info >= (3, 8) and sys.version_info <= (3, 9): - from importlib_metadata import entry_points - # Removed in Python 3.12 - else: - from pkg_resources import iter_entry_points as entry_points + import pkg_resources except ImportError: return - for entrypoint in entry_points(group=group): + for entrypoint in pkg_resources.iter_entry_points(group=group): callback = _create_import_hook_from_entrypoint(entrypoint) register_post_import_hook(callback, entrypoint.name) - # Indicate that a module has been loaded. Any post import hooks which # were registered against the target module will be invoked. If an # exception is raised in any of the post import hooks, that will cause # the import of the target module to fail. - -@synchronized(_post_import_hooks_lock) def notify_module_loaded(module): - name = getattr(module, "__name__", None) - hooks = _post_import_hooks.get(name, None) + name = getattr(module, '__name__', None) - if hooks: - _post_import_hooks[name] = [] + with _post_import_hooks_lock: + hooks = _post_import_hooks.pop(name, ()) - for hook in hooks: - hook(module) + # Note that the hook is called outside of the lock to avoid deadlocks if + # code run as a consequence of calling the module import hook in turn + # triggers a separate thread which tries to register an import hook. + for hook in hooks: + hook(module) # A custom module import finder. This intercepts attempts to import # modules and watches out for attempts to import target modules of # interest. When a module of interest is imported, then any post import # hooks which are registered will be invoked. - class _ImportHookLoader: + def load_module(self, fullname): module = sys.modules[fullname] notify_module_loaded(module) return module +class _ImportHookChainedLoader(ObjectProxy): -class _ImportHookChainedLoader: def __init__(self, loader): - self.loader = loader + super(_ImportHookChainedLoader, self).__init__(loader) if hasattr(loader, "load_module"): - self.load_module = self._load_module + self.__self_setattr__('load_module', self._self_load_module) if hasattr(loader, "create_module"): - self.create_module = self._create_module + self.__self_setattr__('create_module', self._self_create_module) if hasattr(loader, "exec_module"): - self.exec_module = self._exec_module - - def _load_module(self, fullname): - module = self.loader.load_module(fullname) + self.__self_setattr__('exec_module', self._self_exec_module) + + def _self_set_loader(self, module): + # Set module's loader to self.__wrapped__ unless it's already set to + # something else. Import machinery will set it to spec.loader if it is + # None, so handle None as well. The module may not support attribute + # assignment, in which case we simply skip it. Note that we also deal + # with __loader__ not existing at all. This is to future proof things + # due to proposal to remove the attribue as described in the GitHub + # issue at https://github.com/python/cpython/issues/77458. Also prior + # to Python 3.3, the __loader__ attribute was only set if a custom + # module loader was used. It isn't clear whether the attribute still + # existed in that case or was set to None. + + class UNDEFINED: pass + + if getattr(module, "__loader__", UNDEFINED) in (None, self): + try: + module.__loader__ = self.__wrapped__ + except AttributeError: + pass + + if (getattr(module, "__spec__", None) is not None + and getattr(module.__spec__, "loader", None) is self): + module.__spec__.loader = self.__wrapped__ + + def _self_load_module(self, fullname): + module = self.__wrapped__.load_module(fullname) + self._self_set_loader(module) notify_module_loaded(module) return module @@ -187,26 +176,27 @@ def _load_module(self, fullname): # Python 3.4 introduced create_module() and exec_module() instead of # load_module() alone. Splitting the two steps. - def _create_module(self, spec): - return self.loader.create_module(spec) + def _self_create_module(self, spec): + return self.__wrapped__.create_module(spec) - def _exec_module(self, module): - self.loader.exec_module(module) + def _self_exec_module(self, module): + self._self_set_loader(module) + self.__wrapped__.exec_module(module) notify_module_loaded(module) - class ImportHookFinder: + def __init__(self): self.in_progress = {} - @synchronized(_post_import_hooks_lock) def find_module(self, fullname, path=None): # If the module being imported is not one we have registered # post import hooks for, we can return immediately. We will # take no further part in the importing of this module. - if not fullname in _post_import_hooks: - return None + with _post_import_hooks_lock: + if fullname not in _post_import_hooks: + return None # When we are interested in a specific module, we will call back # into the import system a second time to defer to the import @@ -262,8 +252,9 @@ def find_spec(self, fullname, path=None, target=None): # post import hooks for, we can return immediately. We will # take no further part in the importing of this module. - if not fullname in _post_import_hooks: - return None + with _post_import_hooks_lock: + if fullname not in _post_import_hooks: + return None # When we are interested in a specific module, we will call back # into the import system a second time to defer to the import @@ -294,14 +285,11 @@ def find_spec(self, fullname, path=None, target=None): finally: del self.in_progress[fullname] - # Decorator for marking that a function should be called as a post # import hook when the target module is imported. - def when_imported(name): def register(hook): register_post_import_hook(hook, name) return hook - return register diff --git a/newrelic/packages/wrapt/patches.py b/newrelic/packages/wrapt/patches.py new file mode 100644 index 0000000000..e22adf7ca8 --- /dev/null +++ b/newrelic/packages/wrapt/patches.py @@ -0,0 +1,141 @@ +import inspect +import sys + +PY2 = sys.version_info[0] == 2 + +if PY2: + string_types = basestring, +else: + string_types = str, + +from .__wrapt__ import FunctionWrapper + +# Helper functions for applying wrappers to existing functions. + +def resolve_path(module, name): + if isinstance(module, string_types): + __import__(module) + module = sys.modules[module] + + parent = module + + path = name.split('.') + attribute = path[0] + + # We can't just always use getattr() because in doing + # that on a class it will cause binding to occur which + # will complicate things later and cause some things not + # to work. For the case of a class we therefore access + # the __dict__ directly. To cope though with the wrong + # class being given to us, or a method being moved into + # a base class, we need to walk the class hierarchy to + # work out exactly which __dict__ the method was defined + # in, as accessing it from __dict__ will fail if it was + # not actually on the class given. Fallback to using + # getattr() if we can't find it. If it truly doesn't + # exist, then that will fail. + + def lookup_attribute(parent, attribute): + if inspect.isclass(parent): + for cls in inspect.getmro(parent): + if attribute in vars(cls): + return vars(cls)[attribute] + else: + return getattr(parent, attribute) + else: + return getattr(parent, attribute) + + original = lookup_attribute(parent, attribute) + + for attribute in path[1:]: + parent = original + original = lookup_attribute(parent, attribute) + + return (parent, attribute, original) + +def apply_patch(parent, attribute, replacement): + setattr(parent, attribute, replacement) + +def wrap_object(module, name, factory, args=(), kwargs={}): + (parent, attribute, original) = resolve_path(module, name) + wrapper = factory(original, *args, **kwargs) + apply_patch(parent, attribute, wrapper) + return wrapper + +# Function for applying a proxy object to an attribute of a class +# instance. The wrapper works by defining an attribute of the same name +# on the class which is a descriptor and which intercepts access to the +# instance attribute. Note that this cannot be used on attributes which +# are themselves defined by a property object. + +class AttributeWrapper(object): + + def __init__(self, attribute, factory, args, kwargs): + self.attribute = attribute + self.factory = factory + self.args = args + self.kwargs = kwargs + + def __get__(self, instance, owner): + value = instance.__dict__[self.attribute] + return self.factory(value, *self.args, **self.kwargs) + + def __set__(self, instance, value): + instance.__dict__[self.attribute] = value + + def __delete__(self, instance): + del instance.__dict__[self.attribute] + +def wrap_object_attribute(module, name, factory, args=(), kwargs={}): + path, attribute = name.rsplit('.', 1) + parent = resolve_path(module, path)[2] + wrapper = AttributeWrapper(attribute, factory, args, kwargs) + apply_patch(parent, attribute, wrapper) + return wrapper + +# Functions for creating a simple decorator using a FunctionWrapper, +# plus short cut functions for applying wrappers to functions. These are +# for use when doing monkey patching. For a more featured way of +# creating decorators see the decorator decorator instead. + +def function_wrapper(wrapper): + def _wrapper(wrapped, instance, args, kwargs): + target_wrapped = args[0] + if instance is None: + target_wrapper = wrapper + elif inspect.isclass(instance): + target_wrapper = wrapper.__get__(None, instance) + else: + target_wrapper = wrapper.__get__(instance, type(instance)) + return FunctionWrapper(target_wrapped, target_wrapper) + return FunctionWrapper(wrapper, _wrapper) + +def wrap_function_wrapper(module, name, wrapper): + return wrap_object(module, name, FunctionWrapper, (wrapper,)) + +def patch_function_wrapper(module, name, enabled=None): + def _wrapper(wrapper): + return wrap_object(module, name, FunctionWrapper, (wrapper, enabled)) + return _wrapper + +def transient_function_wrapper(module, name): + def _decorator(wrapper): + def _wrapper(wrapped, instance, args, kwargs): + target_wrapped = args[0] + if instance is None: + target_wrapper = wrapper + elif inspect.isclass(instance): + target_wrapper = wrapper.__get__(None, instance) + else: + target_wrapper = wrapper.__get__(instance, type(instance)) + def _execute(wrapped, instance, args, kwargs): + (parent, attribute, original) = resolve_path(module, name) + replacement = FunctionWrapper(original, target_wrapper) + setattr(parent, attribute, replacement) + try: + return wrapped(*args, **kwargs) + finally: + setattr(parent, attribute, original) + return FunctionWrapper(target_wrapped, _execute) + return FunctionWrapper(wrapper, _wrapper) + return _decorator diff --git a/newrelic/packages/wrapt/weakrefs.py b/newrelic/packages/wrapt/weakrefs.py new file mode 100644 index 0000000000..f931b60d5f --- /dev/null +++ b/newrelic/packages/wrapt/weakrefs.py @@ -0,0 +1,98 @@ +import functools +import weakref + +from .__wrapt__ import ObjectProxy, _FunctionWrapperBase + +# A weak function proxy. This will work on instance methods, class +# methods, static methods and regular functions. Special treatment is +# needed for the method types because the bound method is effectively a +# transient object and applying a weak reference to one will immediately +# result in it being destroyed and the weakref callback called. The weak +# reference is therefore applied to the instance the method is bound to +# and the original function. The function is then rebound at the point +# of a call via the weak function proxy. + +def _weak_function_proxy_callback(ref, proxy, callback): + if proxy._self_expired: + return + + proxy._self_expired = True + + # This could raise an exception. We let it propagate back and let + # the weakref.proxy() deal with it, at which point it generally + # prints out a short error message direct to stderr and keeps going. + + if callback is not None: + callback(proxy) + +class WeakFunctionProxy(ObjectProxy): + + __slots__ = ('_self_expired', '_self_instance') + + def __init__(self, wrapped, callback=None): + # We need to determine if the wrapped function is actually a + # bound method. In the case of a bound method, we need to keep a + # reference to the original unbound function and the instance. + # This is necessary because if we hold a reference to the bound + # function, it will be the only reference and given it is a + # temporary object, it will almost immediately expire and + # the weakref callback triggered. So what is done is that we + # hold a reference to the instance and unbound function and + # when called bind the function to the instance once again and + # then call it. Note that we avoid using a nested function for + # the callback here so as not to cause any odd reference cycles. + + _callback = callback and functools.partial( + _weak_function_proxy_callback, proxy=self, + callback=callback) + + self._self_expired = False + + if isinstance(wrapped, _FunctionWrapperBase): + self._self_instance = weakref.ref(wrapped._self_instance, + _callback) + + if wrapped._self_parent is not None: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped._self_parent, _callback)) + + else: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped, _callback)) + + return + + try: + self._self_instance = weakref.ref(wrapped.__self__, _callback) + + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped.__func__, _callback)) + + except AttributeError: + self._self_instance = None + + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped, _callback)) + + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + + # We perform a boolean check here on the instance and wrapped + # function as that will trigger the reference error prior to + # calling if the reference had expired. + + instance = self._self_instance and self._self_instance() + function = self.__wrapped__ and self.__wrapped__ + + # If the wrapped function was originally a bound function, for + # which we retained a reference to the instance and the unbound + # function we need to rebind the function and then call it. If + # not just called the wrapped function. + + if instance is None: + return self.__wrapped__(*args, **kwargs) + + return function.__get__(instance, type(instance))(*args, **kwargs) diff --git a/newrelic/packages/wrapt/wrappers.py b/newrelic/packages/wrapt/wrappers.py index 2716cd1da1..dfc3440db4 100644 --- a/newrelic/packages/wrapt/wrappers.py +++ b/newrelic/packages/wrapt/wrappers.py @@ -1,8 +1,5 @@ -import os import sys -import functools import operator -import weakref import inspect PY2 = sys.version_info[0] == 2 @@ -94,6 +91,9 @@ def __init__(self, wrapped): except AttributeError: pass + def __self_setattr__(self, name, value): + object.__setattr__(self, name, value) + @property def __name__(self): return self.__wrapped__.__name__ @@ -445,12 +445,22 @@ def __reduce_ex__(self, protocol): class CallableObjectProxy(ObjectProxy): - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + return self.__wrapped__(*args, **kwargs) class PartialCallableObjectProxy(ObjectProxy): - def __init__(self, *args, **kwargs): + def __init__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + if len(args) < 1: raise TypeError('partial type takes at least one argument') @@ -464,7 +474,12 @@ def __init__(self, *args, **kwargs): self._self_args = args self._self_kwargs = kwargs - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + _args = self._self_args + args _kwargs = dict(self._self_kwargs) @@ -544,7 +559,12 @@ def __get__(self, instance, owner): return self - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + # If enabled has been specified, then evaluate it at this point # and if the wrapper is not to be executed, then simply return # the bound function rather than a bound wrapper for the bound @@ -607,7 +627,12 @@ def __subclasscheck__(self, subclass): class BoundFunctionWrapper(_FunctionWrapperBase): - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + # If enabled has been specified, then evaluate it at this point # and if the wrapper is not to be executed, then simply return # the bound function rather than a bound wrapper for the bound @@ -757,230 +782,3 @@ def __init__(self, wrapped, wrapper, enabled=None): super(FunctionWrapper, self).__init__(wrapped, None, wrapper, enabled, binding) - -try: - if not os.environ.get('WRAPT_DISABLE_EXTENSIONS'): - from ._wrappers import (ObjectProxy, CallableObjectProxy, - PartialCallableObjectProxy, FunctionWrapper, - BoundFunctionWrapper, _FunctionWrapperBase) -except ImportError: - pass - -# Helper functions for applying wrappers to existing functions. - -def resolve_path(module, name): - if isinstance(module, string_types): - __import__(module) - module = sys.modules[module] - - parent = module - - path = name.split('.') - attribute = path[0] - - # We can't just always use getattr() because in doing - # that on a class it will cause binding to occur which - # will complicate things later and cause some things not - # to work. For the case of a class we therefore access - # the __dict__ directly. To cope though with the wrong - # class being given to us, or a method being moved into - # a base class, we need to walk the class hierarchy to - # work out exactly which __dict__ the method was defined - # in, as accessing it from __dict__ will fail if it was - # not actually on the class given. Fallback to using - # getattr() if we can't find it. If it truly doesn't - # exist, then that will fail. - - def lookup_attribute(parent, attribute): - if inspect.isclass(parent): - for cls in inspect.getmro(parent): - if attribute in vars(cls): - return vars(cls)[attribute] - else: - return getattr(parent, attribute) - else: - return getattr(parent, attribute) - - original = lookup_attribute(parent, attribute) - - for attribute in path[1:]: - parent = original - original = lookup_attribute(parent, attribute) - - return (parent, attribute, original) - -def apply_patch(parent, attribute, replacement): - setattr(parent, attribute, replacement) - -def wrap_object(module, name, factory, args=(), kwargs={}): - (parent, attribute, original) = resolve_path(module, name) - wrapper = factory(original, *args, **kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Function for applying a proxy object to an attribute of a class -# instance. The wrapper works by defining an attribute of the same name -# on the class which is a descriptor and which intercepts access to the -# instance attribute. Note that this cannot be used on attributes which -# are themselves defined by a property object. - -class AttributeWrapper(object): - - def __init__(self, attribute, factory, args, kwargs): - self.attribute = attribute - self.factory = factory - self.args = args - self.kwargs = kwargs - - def __get__(self, instance, owner): - value = instance.__dict__[self.attribute] - return self.factory(value, *self.args, **self.kwargs) - - def __set__(self, instance, value): - instance.__dict__[self.attribute] = value - - def __delete__(self, instance): - del instance.__dict__[self.attribute] - -def wrap_object_attribute(module, name, factory, args=(), kwargs={}): - path, attribute = name.rsplit('.', 1) - parent = resolve_path(module, path)[2] - wrapper = AttributeWrapper(attribute, factory, args, kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Functions for creating a simple decorator using a FunctionWrapper, -# plus short cut functions for applying wrappers to functions. These are -# for use when doing monkey patching. For a more featured way of -# creating decorators see the decorator decorator instead. - -def function_wrapper(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - return FunctionWrapper(target_wrapped, target_wrapper) - return FunctionWrapper(wrapper, _wrapper) - -def wrap_function_wrapper(module, name, wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - -def patch_function_wrapper(module, name): - def _wrapper(wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - return _wrapper - -def transient_function_wrapper(module, name): - def _decorator(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - def _execute(wrapped, instance, args, kwargs): - (parent, attribute, original) = resolve_path(module, name) - replacement = FunctionWrapper(original, target_wrapper) - setattr(parent, attribute, replacement) - try: - return wrapped(*args, **kwargs) - finally: - setattr(parent, attribute, original) - return FunctionWrapper(target_wrapped, _execute) - return FunctionWrapper(wrapper, _wrapper) - return _decorator - -# A weak function proxy. This will work on instance methods, class -# methods, static methods and regular functions. Special treatment is -# needed for the method types because the bound method is effectively a -# transient object and applying a weak reference to one will immediately -# result in it being destroyed and the weakref callback called. The weak -# reference is therefore applied to the instance the method is bound to -# and the original function. The function is then rebound at the point -# of a call via the weak function proxy. - -def _weak_function_proxy_callback(ref, proxy, callback): - if proxy._self_expired: - return - - proxy._self_expired = True - - # This could raise an exception. We let it propagate back and let - # the weakref.proxy() deal with it, at which point it generally - # prints out a short error message direct to stderr and keeps going. - - if callback is not None: - callback(proxy) - -class WeakFunctionProxy(ObjectProxy): - - __slots__ = ('_self_expired', '_self_instance') - - def __init__(self, wrapped, callback=None): - # We need to determine if the wrapped function is actually a - # bound method. In the case of a bound method, we need to keep a - # reference to the original unbound function and the instance. - # This is necessary because if we hold a reference to the bound - # function, it will be the only reference and given it is a - # temporary object, it will almost immediately expire and - # the weakref callback triggered. So what is done is that we - # hold a reference to the instance and unbound function and - # when called bind the function to the instance once again and - # then call it. Note that we avoid using a nested function for - # the callback here so as not to cause any odd reference cycles. - - _callback = callback and functools.partial( - _weak_function_proxy_callback, proxy=self, - callback=callback) - - self._self_expired = False - - if isinstance(wrapped, _FunctionWrapperBase): - self._self_instance = weakref.ref(wrapped._self_instance, - _callback) - - if wrapped._self_parent is not None: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped._self_parent, _callback)) - - else: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - return - - try: - self._self_instance = weakref.ref(wrapped.__self__, _callback) - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped.__func__, _callback)) - - except AttributeError: - self._self_instance = None - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - def __call__(self, *args, **kwargs): - # We perform a boolean check here on the instance and wrapped - # function as that will trigger the reference error prior to - # calling if the reference had expired. - - instance = self._self_instance and self._self_instance() - function = self.__wrapped__ and self.__wrapped__ - - # If the wrapped function was originally a bound function, for - # which we retained a reference to the instance and the unbound - # function we need to rebind the function and then call it. If - # not just called the wrapped function. - - if instance is None: - return self.__wrapped__(*args, **kwargs) - - return function.__get__(instance, type(instance))(*args, **kwargs) diff --git a/tests/agent_streaming/test_infinite_tracing.py b/tests/agent_streaming/test_infinite_tracing.py index f1119c38cd..59060347e1 100644 --- a/tests/agent_streaming/test_infinite_tracing.py +++ b/tests/agent_streaming/test_infinite_tracing.py @@ -389,12 +389,12 @@ def _test(): # Wait for OK status code to close the channel start_time = time.time() while not (request_iterator._stream and request_iterator._stream.done()): - assert time.time() - start_time < 5, "Timed out waiting for OK status code." + assert time.time() - start_time < 15, "Timed out waiting for OK status code." time.sleep(0.5) # Put new span and wait until buffer has been emptied and either sent or lost stream_buffer.put(span) - assert spans_processed_event.wait(timeout=5), "Data lost in stream buffer iterator." + assert spans_processed_event.wait(timeout=15), "Data lost in stream buffer iterator." _test() diff --git a/tests/agent_unittests/test_wrappers.py b/tests/agent_unittests/test_wrappers.py new file mode 100644 index 0000000000..eccee4df5b --- /dev/null +++ b/tests/agent_unittests/test_wrappers.py @@ -0,0 +1,81 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.common.object_wrapper import function_wrapper + + +@pytest.fixture(scope="function") +def wrapper(): + @function_wrapper + def _wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) + + return _wrapper + + +@pytest.fixture(scope="function") +def wrapped_function(wrapper): + @wrapper + def wrapped(): + return True + + return wrapped + + +def test_nr_prefix_attributes(wrapped_function): + wrapped_function._nr_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._nr_attr == 1, "_nr_ attributes should be stored on wrapper object and retrievable." + assert "_nr_attr" not in vars_, "_nr_ attributes should NOT appear in __dict__." + + +def test_self_prefix_attributes(wrapped_function): + wrapped_function._self_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._self_attr == 1, "_self_ attributes should be stored on wrapper object and retrievable." + assert "_nr_attr" not in vars_, "_self_ attributes should NOT appear in __dict__." + + +def test_prefixed_attributes_share_namespace(wrapped_function): + wrapped_function._nr_attr = 1 + wrapped_function._self_attr = 2 + + assert ( + wrapped_function._nr_attr == 2 + ), "_nr_ attributes share a namespace with _self_ attributes and should be overwritten." + + +def test_wrapped_function_attributes(wrapped_function): + wrapped_function._other_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._other_attr == 1, "All other attributes should be stored on wrapped object and retrievable." + assert "_other_attr" in vars_, "Other types of attributes SHOULD appear in __dict__." + + assert wrapped_function() + + +def test_multiple_wrapper_last_object(wrapper): + def wrapped(): + pass + + wrapper_1 = wrapper(wrapped) + wrapper_2 = wrapper(wrapper_1) + + assert wrapper_2._nr_last_object is wrapped, "Last object in chain should be the wrapped function." + assert wrapper_2._nr_next_object is wrapper_1, "Next object in chain should be the middle function."