heckyel/yt-local
0
0
Fork 0
Code Issues Pull Requests Releases Activity

Windows: Use 32-bit distribution of python

Browse Source
This commit is contained in:
James Taylor
2018-09-14 19:32:27 -07:00
parent 6ca20ff701
commit 4212164e91
166 changed files with 175548 additions and 44620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

530
python/gevent/local.py
View File

@@ -1,3 +1,4 @@
# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
"""
Greenlet-local objects.
@@ -133,161 +134,472 @@ affects what we see:
This results in locals being eligible for garbage collection as soon
as their greenlet exits.
.. versionchanged:: 1.2.3
Use a weak-reference to clear the greenlet link we establish in case
the local object dies before the greenlet does.
.. versionchanged:: 1.3a1
Implement the methods for attribute access directly, handling
descriptors directly here. This allows removing the use of a lock
and facilitates greatly improved performance.
.. versionchanged:: 1.3a1
The ``__init__`` method of subclasses of ``local`` is no longer
called with a lock held. CPython does not use such a lock in its
native implementation. This could potentially show as a difference
if code that uses multiple dependent attributes in ``__slots__``
(which are shared across all greenlets) switches during ``__init__``.
"""
from __future__ import print_function
from copy import copy
from weakref import ref
from contextlib import contextmanager
from gevent.hub import getcurrent
from gevent._compat import PYPY
from gevent.lock import RLock
__all__ = ["local"]
locals()['getcurrent'] = __import__('greenlet').getcurrent
locals()['greenlet_init'] = lambda: None
__all__ = [
"local",
]
# The key used in the Thread objects' attribute dicts.
# We keep it a string for speed but make it unlikely to clash with
# a "real" attribute.
key_prefix = '_gevent_local_localimpl_'
# The overall structure is as follows:
# For each local() object:
# greenlet.__dict__[key_prefix + str(id(local))]
# => _localimpl.dicts[id(greenlet)] => (ref(greenlet), {})
# That final tuple is actually a localimpl_dict_entry object.
def all_local_dicts_for_greenlet(greenlet):
"""
Internal debug helper for getting the local values associated
with a greenlet. This is subject to change or removal at any time.
:return: A list of ((type, id), {}) pairs, where the first element
is the type and id of the local object and the second object is its
instance dictionary, as seen from this greenlet.
.. versionadded:: 1.3a2
"""
result = []
id_greenlet = id(greenlet)
greenlet_dict = greenlet.__dict__
for k, v in greenlet_dict.items():
if not k.startswith(key_prefix):
continue
local_impl = v()
if local_impl is None:
continue
entry = local_impl.dicts.get(id_greenlet)
if entry is None:
# Not yet used in this greenlet.
continue
assert entry.wrgreenlet() is greenlet
result.append((local_impl.localtypeid, entry.localdict))
return result
class _wrefdict(dict):
"""A dict that can be weak referenced"""
class _greenlet_deleted(object):
"""
A weakref callback for when the greenlet
is deleted.
If the greenlet is a `gevent.greenlet.Greenlet` and
supplies ``rawlink``, that will be used instead of a
weakref.
"""
__slots__ = ('idt', 'wrdicts')
def __init__(self, idt, wrdicts):
self.idt = idt
self.wrdicts = wrdicts
def __call__(self, _unused):
dicts = self.wrdicts()
if dicts:
dicts.pop(self.idt, None)
class _local_deleted(object):
__slots__ = ('key', 'wrthread', 'greenlet_deleted')
def __init__(self, key, wrthread, greenlet_deleted):
self.key = key
self.wrthread = wrthread
self.greenlet_deleted = greenlet_deleted
def __call__(self, _unused):
thread = self.wrthread()
if thread is not None:
try:
unlink = thread.unlink
except AttributeError:
pass
else:
unlink(self.greenlet_deleted)
del thread.__dict__[self.key]
class _localimpl(object):
"""A class managing thread-local dicts"""
__slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__'
__slots__ = ('key', 'dicts',
'localargs', 'localkwargs',
'localtypeid',
'__weakref__',)
def __init__(self):
# The key used in the Thread objects' attribute dicts.
# We keep it a string for speed but make it unlikely to clash with
# a "real" attribute.
self.key = '_threading_local._localimpl.' + str(id(self))
# { id(Thread) -> (ref(Thread), thread-local dict) }
def __init__(self, args, kwargs, local_type, id_local):
self.key = key_prefix + str(id(self))
# { id(greenlet) -> _localimpl_dict_entry(ref(greenlet), greenlet-local dict) }
self.dicts = _wrefdict()
self.localargs = args
self.localkwargs = kwargs
self.localtypeid = local_type, id_local
def get_dict(self):
"""Return the dict for the current thread. Raises KeyError if none
defined."""
thread = getcurrent()
return self.dicts[id(thread)][1]
# We need to create the thread dict in anticipation of
# __init__ being called, to make sure we don't call it
# again ourselves. MUST do this before setting any attributes.
greenlet = getcurrent() # pylint:disable=undefined-variable
_localimpl_create_dict(self, greenlet, id(greenlet))
def create_dict(self):
"""Create a new dict for the current thread, and return it."""
localdict = {}
key = self.key
thread = getcurrent()
idt = id(thread)
class _localimpl_dict_entry(object):
"""
The object that goes in the ``dicts`` of ``_localimpl``
object for each thread.
"""
# This is a class, not just a tuple, so that cython can optimize
# attribute access
__slots__ = ('wrgreenlet', 'localdict')
# If we are working with a gevent.greenlet.Greenlet, we can
# pro-actively clear out with a link. Use rawlink to avoid
# spawning any more greenlets
try:
rawlink = thread.rawlink
except AttributeError:
# Otherwise we need to do it with weak refs
def local_deleted(_, key=key):
# When the localimpl is deleted, remove the thread attribute.
thread = wrthread()
if thread is not None:
del thread.__dict__[key]
def __init__(self, wrgreenlet, localdict):
self.wrgreenlet = wrgreenlet
self.localdict = localdict
def thread_deleted(_, idt=idt):
# When the thread is deleted, remove the local dict.
# Note that this is suboptimal if the thread object gets
# caught in a reference loop. We would like to be called
# as soon as the OS-level thread ends instead.
_local = wrlocal()
if _local is not None:
_local.dicts.pop(idt, None)
wrlocal = ref(self, local_deleted)
wrthread = ref(thread, thread_deleted)
thread.__dict__[key] = wrlocal
else:
wrdicts = ref(self.dicts)
def clear(_):
dicts = wrdicts()
if dicts:
dicts.pop(idt, None)
rawlink(clear)
wrthread = None
self.dicts[idt] = wrthread, localdict
return localdict
# We use functions instead of methods so that they can be cdef'd in
# local.pxd; if they were cdef'd as methods, they would cause
# the creation of a pointer and a vtable. This happens
# even if we declare the class @cython.final. functions thus save memory overhead
# (but not pointer chasing overhead; the vtable isn't used when we declare
# the class final).
@contextmanager
def _patch(self):
impl = object.__getattribute__(self, '_local__impl')
orig_dct = object.__getattribute__(self, '__dict__')
def _localimpl_create_dict(self, greenlet, id_greenlet):
"""Create a new dict for the current thread, and return it."""
localdict = {}
key = self.key
wrdicts = ref(self.dicts)
# When the greenlet is deleted, remove the local dict.
# Note that this is suboptimal if the greenlet object gets
# caught in a reference loop. We would like to be called
# as soon as the OS-level greenlet ends instead.
# If we are working with a gevent.greenlet.Greenlet, we
# can pro-actively clear out with a link, avoiding the
# issue described above. Use rawlink to avoid spawning any
# more greenlets.
greenlet_deleted = _greenlet_deleted(id_greenlet, wrdicts)
rawlink = getattr(greenlet, 'rawlink', None)
if rawlink is not None:
rawlink(greenlet_deleted)
wrthread = ref(greenlet)
else:
wrthread = ref(greenlet, greenlet_deleted)
# When the localimpl is deleted, remove the thread attribute.
local_deleted = _local_deleted(key, wrthread, greenlet_deleted)
wrlocal = ref(self, local_deleted)
greenlet.__dict__[key] = wrlocal
self.dicts[id_greenlet] = _localimpl_dict_entry(wrthread, localdict)
return localdict
_marker = object()
def _local_get_dict(self):
impl = self._local__impl
# Cython can optimize dict[], but not dict.get()
greenlet = getcurrent() # pylint:disable=undefined-variable
idg = id(greenlet)
try:
dct = impl.get_dict()
entry = impl.dicts[idg]
dct = entry.localdict
except KeyError:
# it's OK to acquire the lock here and not earlier, because the above code won't switch out
# however, subclassed __init__ might switch, so we do need to acquire the lock here
dct = impl.create_dict()
args, kw = impl.localargs
with impl.locallock:
self.__init__(*args, **kw)
with impl.locallock:
object.__setattr__(self, '__dict__', dct)
yield
object.__setattr__(self, '__dict__', orig_dct)
dct = _localimpl_create_dict(impl, greenlet, idg)
self.__init__(*impl.localargs, **impl.localkwargs)
return dct
def _init():
greenlet_init() # pylint:disable=undefined-variable
_local_attrs = {
'_local__impl',
'_local_type_get_descriptors',
'_local_type_set_or_del_descriptors',
'_local_type_del_descriptors',
'_local_type_set_descriptors',
'_local_type',
'_local_type_vars',
'__class__',
'__cinit__',
}
class local(object):
"""
An object whose attributes are greenlet-local.
"""
__slots__ = '_local__impl', '__dict__'
__slots__ = tuple(_local_attrs - {'__class__', '__cinit__'})
def __new__(cls, *args, **kw):
def __cinit__(self, *args, **kw):
if args or kw:
if (PYPY and cls.__init__ == object.__init__) or (not PYPY and cls.__init__ is object.__init__):
raise TypeError("Initialization arguments are not supported")
self = object.__new__(cls)
impl = _localimpl()
impl.localargs = (args, kw)
impl.locallock = RLock()
object.__setattr__(self, '_local__impl', impl)
# We need to create the thread dict in anticipation of
# __init__ being called, to make sure we don't call it
# again ourselves.
impl.create_dict()
return self
if type(self).__init__ == object.__init__:
raise TypeError("Initialization arguments are not supported", args, kw)
impl = _localimpl(args, kw, type(self), id(self))
# pylint:disable=attribute-defined-outside-init
self._local__impl = impl
get, dels, sets_or_dels, sets = _local_find_descriptors(self)
self._local_type_get_descriptors = get
self._local_type_set_or_del_descriptors = sets_or_dels
self._local_type_del_descriptors = dels
self._local_type_set_descriptors = sets
self._local_type = type(self)
self._local_type_vars = set(dir(self._local_type))
def __getattribute__(self, name):
with _patch(self):
def __getattribute__(self, name): # pylint:disable=too-many-return-statements
if name in _local_attrs:
# The _local__impl, __cinit__, etc, won't be hit by the
# Cython version, if we've done things right. If we haven't,
# they will be, and this will produce an error.
return object.__getattribute__(self, name)
dct = _local_get_dict(self)
if name == '__dict__':
return dct
# If there's no possible way we can switch, because this
# attribute is *not* found in the class where it might be a
# data descriptor (property), and it *is* in the dict
# then we don't need to swizzle the dict and take the lock.
# We don't have to worry about people overriding __getattribute__
# because if they did, the dict-swizzling would only last as
# long as we were in here anyway.
# Similarly, a __getattr__ will still be called by _oga() if needed
# if it's not in the dict.
# Optimization: If we're not subclassed, then
# there can be no descriptors except for methods, which will
# never need to use __dict__.
if self._local_type is local:
return dct[name] if name in dct else object.__getattribute__(self, name)
# NOTE: If this is a descriptor, this will invoke its __get__.
# A broken descriptor that doesn't return itself when called with
# a None for the instance argument could mess us up here.
# But this is faster than a loop over mro() checking each class __dict__
# manually.
if name in dct:
if name not in self._local_type_vars:
# If there is a dict value, and nothing in the type,
# it can't possibly be a descriptor, so it is just returned.
return dct[name]
# It's in the type *and* in the dict. If the type value is
# a data descriptor (defines __get__ *and* either __set__ or
# __delete__), then the type wins. If it's a non-data descriptor
# (defines just __get__), then the instance wins. If it's not a
# descriptor at all (doesn't have __get__), the instance wins.
# NOTE that the docs for descriptors say that these methods must be
# defined on the *class* of the object in the type.
if name not in self._local_type_get_descriptors:
# Entirely not a descriptor. Instance wins.
return dct[name]
if name in self._local_type_set_or_del_descriptors:
# A data descriptor.
# arbitrary code execution while these run. If they touch self again,
# they'll call back into us and we'll repeat the dance.
type_attr = getattr(self._local_type, name)
return type(type_attr).__get__(type_attr, self, self._local_type)
# Last case is a non-data descriptor. Instance wins.
return dct[name]
if name in self._local_type_vars:
# Not in the dictionary, but is found in the type. It could be
# a non-data descriptor still. Some descriptors, like @staticmethod,
# return objects (functions, in this case), that are *themselves*
# descriptors, which when invoked, again, would do the wrong thing.
# So we can't rely on getattr() on the type for them, we have to
# look through the MRO dicts ourself.
if name not in self._local_type_get_descriptors:
# Not a descriptor, can't execute code. So all we need is
# the return value of getattr() on our type.
return getattr(self._local_type, name)
for base in self._local_type.mro():
bd = base.__dict__
if name in bd:
attr_on_type = bd[name]
result = type(attr_on_type).__get__(attr_on_type, self, self._local_type)
return result
# It wasn't in the dict and it wasn't in the type.
# So the next step is to invoke type(self)__getattr__, if it
# exists, otherwise raise an AttributeError.
# we will invoke type(self).__getattr__ or raise an attribute error.
if hasattr(self._local_type, '__getattr__'):
return self._local_type.__getattr__(self, name)
raise AttributeError("%r object has no attribute '%s'"
% (self._local_type.__name__, name))
def __setattr__(self, name, value):
if name == '__dict__':
raise AttributeError(
"%r object attribute '__dict__' is read-only"
% self.__class__.__name__)
with _patch(self):
return object.__setattr__(self, name, value)
% type(self))
if name in _local_attrs:
object.__setattr__(self, name, value)
return
dct = _local_get_dict(self)
if self._local_type is local:
# Optimization: If we're not subclassed, we can't
# have data descriptors, so this goes right in the dict.
dct[name] = value
return
if name in self._local_type_vars:
if name in self._local_type_set_descriptors:
type_attr = getattr(self._local_type, name, _marker)
# A data descriptor, like a property or a slot.
type(type_attr).__set__(type_attr, self, value)
return
# Otherwise it goes directly in the dict
dct[name] = value
def __delattr__(self, name):
if name == '__dict__':
raise AttributeError(
"%r object attribute '__dict__' is read-only"
% self.__class__.__name__)
with _patch(self):
return object.__delattr__(self, name)
if name in self._local_type_vars:
if name in self._local_type_del_descriptors:
# A data descriptor, like a property or a slot.
type_attr = getattr(self._local_type, name, _marker)
type(type_attr).__delete__(type_attr, self)
return
# Otherwise it goes directly in the dict
# Begin inlined function _get_dict()
dct = _local_get_dict(self)
try:
del dct[name]
except KeyError:
raise AttributeError(name)
def __copy__(self):
impl = object.__getattribute__(self, '_local__impl')
current = getcurrent()
currentId = id(current)
d = impl.get_dict()
duplicate = copy(d)
impl = self._local__impl
entry = impl.dicts[id(getcurrent())] # pylint:disable=undefined-variable
dct = entry.localdict
duplicate = copy(dct)
cls = type(self)
if (PYPY and cls.__init__ != object.__init__) or (not PYPY and cls.__init__ is not object.__init__):
args, kw = impl.localargs
instance = cls(*args, **kw)
else:
instance = cls()
new_impl = object.__getattribute__(instance, '_local__impl')
tpl = new_impl.dicts[currentId]
new_impl.dicts[currentId] = (tpl[0], duplicate)
instance = cls(*impl.localargs, **impl.localkwargs)
_local__copy_dict_from(instance, impl, duplicate)
return instance
def _local__copy_dict_from(self, impl, duplicate):
current = getcurrent() # pylint:disable=undefined-variable
currentId = id(current)
new_impl = self._local__impl
assert new_impl is not impl
entry = new_impl.dicts[currentId]
new_impl.dicts[currentId] = _localimpl_dict_entry(entry.wrgreenlet, duplicate)
def _local_find_descriptors(self):
type_self = type(self)
gets = set()
dels = set()
set_or_del = set()
sets = set()
mro = list(type_self.mro())
for attr_name in dir(type_self):
# Conventionally, descriptors when called on a class
# return themself, but not all do. Notable exceptions are
# in the zope.interface package, where things like __provides__
# return other class attributes. So we can't use getattr, and instead
# walk up the dicts
for base in mro:
bd = base.__dict__
if attr_name in bd:
attr = bd[attr_name]
break
else:
raise AttributeError(attr_name)
type_attr = type(attr)
if hasattr(type_attr, '__get__'):
gets.add(attr_name)
if hasattr(type_attr, '__delete__'):
dels.add(attr_name)
set_or_del.add(attr_name)
if hasattr(type_attr, '__set__'):
sets.add(attr_name)
return (gets, dels, set_or_del, sets)
# Cython doesn't let us use __new__, it requires
# __cinit__. But we need __new__ if we're not compiled
# (e.g., on PyPy). So we set it at runtime. Cython
# will raise an error if we're compiled.
def __new__(cls, *args, **kw):
self = super(local, cls).__new__(cls)
# We get the cls in *args for some reason
# too when we do it this way....except on PyPy3, which does
# not *unless* it's wrapped in a classmethod (which it is)
self.__cinit__(*args[1:], **kw)
return self
try:
# PyPy2/3 and CPython handle adding a __new__ to the class
# in different ways. In CPython and PyPy3, it must be wrapped with classmethod;
# in PyPy2, it must not. In either case, the args that get passed to
# it are stil wrong.
local.__new__ = 'None'
except TypeError: # pragma: no cover
# Must be compiled
pass
else:
from gevent._compat import PYPY
from gevent._compat import PY2
if PYPY and PY2:
local.__new__ = __new__
else:
local.__new__ = classmethod(__new__)
del PYPY
del PY2
_init()
from gevent._util import import_c_accel
import_c_accel(globals(), 'gevent._local')