Windows: Use 32-bit distribution of python

python/gevent/__greenlet_primitives.pxd

@@ -0,0 +1,44 @@
+cimport cython
+
+# This file must not cimport anything from gevent.
+
+cdef wref
+
+cdef BlockingSwitchOutError
+
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    object PyGreenlet_Switch(greenlet self, void* args, void* kwargs)
+    void PyGreenlet_Import()
+
+@cython.final
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef bint _greenlet_imported
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+cdef inline object _greenlet_switch(greenlet self):
+    return PyGreenlet_Switch(self, NULL, NULL)
+
+cdef class TrackedRawGreenlet(greenlet):
+    pass
+
+cdef class SwitchOutGreenletWithLoop(TrackedRawGreenlet):
+    cdef public loop
+
+    cpdef switch(self)
+    cpdef switch_out(self)

python/gevent/__hub_local.pxd

@@ -0,0 +1,17 @@
+from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop
+
+cdef _threadlocal
+
+cpdef get_hub_class()
+cpdef SwitchOutGreenletWithLoop get_hub_if_exists()
+cpdef set_hub(SwitchOutGreenletWithLoop hub)
+cpdef get_loop()
+cpdef set_loop(loop)
+
+# We can't cdef this, it won't do varargs.
+# cpdef WaitOperationsGreenlet get_hub(*args, **kwargs)
+
+# XXX: TODO: Move the definition of TrackedRawGreenlet
+# into a file that can be cython compiled so get_hub can
+# return that.
+cpdef SwitchOutGreenletWithLoop get_hub_noargs()

python/gevent/__hub_primitives.pxd

@@ -0,0 +1,69 @@
+cimport cython
+
+from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop
+from gevent.__hub_local cimport get_hub_noargs as get_hub
+
+from gevent.__waiter cimport Waiter
+from gevent.__waiter cimport MultipleWaiter
+
+cdef InvalidSwitchError
+cdef _waiter
+cdef _greenlet_primitives
+cdef traceback
+cdef _timeout_error
+cdef Timeout
+
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+@cython.final
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef bint _greenlet_imported
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+
+cdef class WaitOperationsGreenlet(SwitchOutGreenletWithLoop):
+
+    cpdef wait(self, watcher)
+    cpdef cancel_wait(self, watcher, error, close_watcher=*)
+    cpdef _cancel_wait(self, watcher, error, close_watcher)
+
+cdef class _WaitIterator:
+    cdef SwitchOutGreenletWithLoop _hub
+    cdef MultipleWaiter _waiter
+    cdef _switch
+    cdef _timeout
+    cdef _objects
+    cdef _timer
+    cdef Py_ssize_t _count
+    cdef bint _begun
+
+
+
+    cdef _cleanup(self)
+
+cpdef iwait_on_objects(objects, timeout=*, count=*)
+cpdef wait_on_objects(objects=*, timeout=*, count=*)
+
+cdef _primitive_wait(watcher, timeout, timeout_exc, WaitOperationsGreenlet hub)
+cpdef wait_on_watcher(watcher, timeout=*, timeout_exc=*, WaitOperationsGreenlet hub=*)
+cpdef wait_read(fileno, timeout=*, timeout_exc=*)
+cpdef wait_write(fileno, timeout=*, timeout_exc=*, event=*)
+cpdef wait_readwrite(fileno, timeout=*, timeout_exc=*, event=*)
+cpdef wait_on_socket(socket, watcher, timeout_exc=*)

python/gevent/__ident.pxd

@@ -0,0 +1,26 @@
+cimport cython
+
+cdef extern from "Python.h":
+
+    ctypedef class weakref.ref [object PyWeakReference]:
+        pass
+
+cdef heappop
+cdef heappush
+cdef object WeakKeyDictionary
+cdef type ref
+
+@cython.internal
+@cython.final
+cdef class ValuedWeakRef(ref):
+    cdef object value
+
+@cython.final
+cdef class IdentRegistry:
+    cdef object _registry
+    cdef list _available_idents
+
+    @cython.final
+    cpdef object get_ident(self, obj)
+    @cython.final
+    cpdef _return_ident(self, ValuedWeakRef ref)

python/gevent/__imap.pxd

@@ -0,0 +1,45 @@
+cimport cython
+from gevent._greenlet cimport Greenlet
+from gevent.__semaphore cimport Semaphore
+from gevent._queue cimport UnboundQueue
+
+@cython.freelist(100)
+@cython.internal
+@cython.final
+cdef class Failure:
+    cdef readonly exc
+    cdef raise_exception
+
+cdef inline _raise_exc(Failure failure)
+
+cdef class IMapUnordered(Greenlet):
+    cdef bint _zipped
+    cdef func
+    cdef iterable
+    cdef spawn
+    cdef Semaphore _result_semaphore
+    cdef int _outstanding_tasks
+    cdef int _max_index
+
+    cdef readonly UnboundQueue queue
+    cdef readonly bint finished
+
+    cdef _inext(self)
+    cdef _ispawn(self, func, item, int item_index)
+
+    # Passed to greenlet.link
+    cpdef _on_result(self, greenlet)
+    # Called directly
+    cdef _on_finish(self, exception)
+
+    cdef _iqueue_value_for_success(self, greenlet)
+    cdef _iqueue_value_for_failure(self, greenlet)
+    cdef _iqueue_value_for_self_finished(self)
+    cdef _iqueue_value_for_self_failure(self, exception)
+
+cdef class IMap(IMapUnordered):
+    cdef int index
+    cdef dict _results
+
+    @cython.locals(index=int)
+    cdef _inext(self)

python/gevent/__init__.py

@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
 to provide a high-level synchronous API on top of libev event loop.
 
 See http://www.gevent.org/ for the documentation.
+
+.. versionchanged:: 1.3a2
+   Add the `config` object.
 """
 
 from __future__ import absolute_import
@@ -15,31 +18,43 @@ _version_info = namedtuple('version_info',
 
 #: The programatic version identifier. The fields have (roughly) the
 #: same meaning as :data:`sys.version_info`
-#: Deprecated in 1.2.
-version_info = _version_info(1, 2, 2, 'dev', 0)
+#: .. deprecated:: 1.2
+#:  Use ``pkg_resources.parse_version(__version__)`` (or the equivalent
+#:  ``packaging.version.Version(__version__)``).
+version_info = _version_info(1, 3, 0, 'dev', 0)
 
-#: The human-readable PEP 440 version identifier
-__version__ = '1.2.2'
+#: The human-readable PEP 440 version identifier.
+#: Use ``pkg_resources.parse_version(__version__)`` or
+#: ``packaging.version.Version(__version__)`` to get a machine-usable
+#: value.
+__version__ = '1.3.6'
 
 
-__all__ = ['get_hub',
-           'Greenlet',
-           'GreenletExit',
-           'spawn',
-           'spawn_later',
-           'spawn_raw',
-           'iwait',
-           'wait',
-           'killall',
-           'Timeout',
-           'with_timeout',
-           'getcurrent',
-           'sleep',
-           'idle',
-           'kill',
-           'signal',
-           'fork',
-           'reinit']
+__all__ = [
+    'get_hub',
+    'Greenlet',
+    'GreenletExit',
+    'spawn',
+    'spawn_later',
+    'spawn_raw',
+    'iwait',
+    'wait',
+    'killall',
+    'Timeout',
+    'with_timeout',
+    'getcurrent',
+    'sleep',
+    'idle',
+    'kill',
+    'signal', # deprecated
+    'signal_handler',
+    'fork',
+    'reinit',
+    'getswitchinterval',
+    'setswitchinterval',
+    # Added in 1.3a2
+    'config',
+]
 
 
 import sys
@@ -48,11 +63,37 @@ if sys.platform == 'win32':
     import socket  # pylint:disable=unused-import,useless-suppression
     del socket
 
-from gevent.hub import get_hub, iwait, wait
+try:
+    # Floating point number, in number of seconds,
+    # like time.time
+    getswitchinterval = sys.getswitchinterval
+    setswitchinterval = sys.setswitchinterval
+except AttributeError:
+    # Running on Python 2
+    _switchinterval = 0.005
+
+    def getswitchinterval():
+        return _switchinterval
+
+    def setswitchinterval(interval):
+        # Weed out None and non-numbers. This is not
+        # exactly exception compatible with the Python 3
+        # versions.
+        if interval > 0:
+            global _switchinterval
+            _switchinterval = interval
+
+from gevent._config import config
+from gevent._hub_local import get_hub
+from gevent._hub_primitives import iwait_on_objects as iwait
+from gevent._hub_primitives import wait_on_objects as wait
+
 from gevent.greenlet import Greenlet, joinall, killall
 joinall = joinall # export for pylint
 spawn = Greenlet.spawn
 spawn_later = Greenlet.spawn_later
+#: The singleton configuration object for gevent.
+config = config
 
 from gevent.timeout import Timeout, with_timeout
 from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit
@@ -66,6 +107,7 @@ except ImportError:
 # to treat 'from gevent import signal' as a callable, to matter whether
 # the 'gevent.signal' module has been imported first
 from gevent.hub import signal as _signal_class
+signal_handler = _signal_class
 from gevent import signal as _signal_module
 
 # The object 'gevent.signal' must:

python/gevent/__semaphore.pxd

@@ -1,6 +1,37 @@
+cimport cython
+
+from gevent.__hub_local cimport get_hub_noargs as get_hub
+cdef Timeout
+
+cdef bint _greenlet_imported
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+
+cdef void _init()
+
+
 cdef class Semaphore:
     cdef public int counter
-    cdef readonly object _links
+    cdef readonly list _links
     cdef readonly object _notifier
     cdef public int _dirty
     cdef object __weakref__

python/gevent/__tracer.pxd

@@ -0,0 +1,43 @@
+cimport cython
+
+cdef sys
+cdef traceback
+
+cdef settrace
+cdef getcurrent
+
+cdef format_run_info
+
+cdef perf_counter
+cdef gmctime
+
+
+cdef class GreenletTracer:
+    cpdef readonly object active_greenlet
+    cpdef readonly object previous_trace_function
+    cpdef readonly Py_ssize_t greenlet_switch_counter
+
+    cdef bint _killed
+
+    cpdef _trace(self, str event, tuple args)
+
+    @cython.locals(did_switch=bint)
+    cpdef did_block_hub(self, hub)
+
+    cpdef kill(self)
+
+@cython.internal
+cdef class _HubTracer(GreenletTracer):
+    cpdef readonly object hub
+    cpdef readonly double max_blocking_time
+
+
+cdef class HubSwitchTracer(_HubTracer):
+    cpdef readonly double last_entered_hub
+
+cdef class MaxSwitchTracer(_HubTracer):
+    cpdef readonly double max_blocking
+    cpdef readonly double last_switch
+
+    @cython.locals(switched_at=double)
+    cpdef _trace(self, str event, tuple args)

python/gevent/__waiter.pxd

@@ -0,0 +1,48 @@
+cimport cython
+
+from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop
+from gevent.__hub_local cimport get_hub_noargs as get_hub
+
+cdef sys
+cdef ConcurrentObjectUseError
+
+
+cdef bint _greenlet_imported
+cdef _NONE
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+cdef class Waiter:
+    cdef readonly SwitchOutGreenletWithLoop hub
+    cdef readonly greenlet greenlet
+    cdef readonly value
+    cdef _exception
+
+    cpdef get(self)
+    cpdef clear(self)
+
+    # cpdef of switch leads to parameter errors...
+    #cpdef switch(self, value)
+
+@cython.final
+@cython.internal
+cdef class MultipleWaiter(Waiter):
+    cdef list _values

python/gevent/_compat.py

@@ -5,12 +5,19 @@ internal gevent python 2/python 3 bridges. Not for external use.
 
 from __future__ import print_function, absolute_import, division
 
-
 import sys
+import os
+
 
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] >= 3
 PYPY = hasattr(sys, 'pypy_version_info')
+WIN = sys.platform.startswith("win")
+LINUX = sys.platform.startswith('linux')
+OSX = sys.platform == 'darwin'
+
+
+PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')
 
 ## Types
 
@@ -18,13 +25,20 @@ if PY3:
     string_types = (str,)
     integer_types = (int,)
     text_type = str
+    native_path_types = (str, bytes)
+    thread_mod_name = '_thread'
 
 else:
     import __builtin__ # pylint:disable=import-error
-    string_types = __builtin__.basestring,
+    string_types = (__builtin__.basestring,)
     text_type = __builtin__.unicode
     integer_types = (int, __builtin__.long)
+    native_path_types = string_types
+    thread_mod_name = 'thread'
 
+def NativeStrIO():
+    import io
+    return io.BytesIO() if str is bytes else io.StringIO()
 
 ## Exceptions
 if PY3:
@@ -32,17 +46,115 @@ if PY3:
         if value.__traceback__ is not tb and tb is not None:
             raise value.with_traceback(tb)
         raise value
+    def exc_clear():
+        pass
 
 else:
     from gevent._util_py2 import reraise # pylint:disable=import-error,no-name-in-module
     reraise = reraise # export
+    exc_clear = sys.exc_clear
+
+## import locks
+try:
+    # In Python 3.4 and newer in CPython and PyPy3,
+    # imp.acquire_lock and imp.release_lock are delegated to
+    # '_imp'. (Which is also used by importlib.) 'imp' itself is
+    # deprecated. Avoid that warning.
+    import _imp as imp
+except ImportError:
+    import imp
+imp_acquire_lock = imp.acquire_lock
+imp_release_lock = imp.release_lock
 
 ## Functions
 if PY3:
     iteritems = dict.items
     itervalues = dict.values
     xrange = range
+    izip = zip
+
 else:
     iteritems = dict.iteritems # python 3: pylint:disable=no-member
     itervalues = dict.itervalues # python 3: pylint:disable=no-member
     xrange = __builtin__.xrange
+    from itertools import izip # python 3: pylint:disable=no-member,no-name-in-module
+    izip = izip
+
+# fspath from 3.6 os.py, but modified to raise the same exceptions as the
+# real native implementation.
+# Define for testing
+def _fspath(path):
+    """
+    Return the path representation of a path-like object.
+
+    If str or bytes is passed in, it is returned unchanged. Otherwise the
+    os.PathLike interface is used to get the path representation. If the
+    path representation is not str or bytes, TypeError is raised. If the
+    provided path is not str, bytes, or os.PathLike, TypeError is raised.
+    """
+    if isinstance(path, native_path_types):
+        return path
+
+    # Work from the object's type to match method resolution of other magic
+    # methods.
+    path_type = type(path)
+    try:
+        path_type_fspath = path_type.__fspath__
+    except AttributeError:
+        raise TypeError("expected str, bytes or os.PathLike object, "
+                        "not " + path_type.__name__)
+
+    path_repr = path_type_fspath(path)
+    if isinstance(path_repr, native_path_types):
+        return path_repr
+
+    raise TypeError("expected {}.__fspath__() to return str or bytes, "
+                    "not {}".format(path_type.__name__,
+                                    type(path_repr).__name__))
+try:
+    from os import fspath # pylint: disable=unused-import,no-name-in-module
+except ImportError:
+    # if not available, use the Python version as transparently as
+    # possible
+    fspath = _fspath
+    fspath.__name__ = 'fspath'
+
+try:
+    from os import fsencode # pylint: disable=unused-import,no-name-in-module
+except ImportError:
+    encoding = sys.getfilesystemencoding() or ('utf-8' if not WIN else 'mbcs')
+    errors = 'strict' if WIN and encoding == 'mbcs' else 'surrogateescape'
+
+    # Added in 3.2, so this is for Python 2.7. Note that it doesn't have
+    # sys.getfilesystemencodeerrors(), which was added in 3.6
+    def fsencode(filename):
+        """Encode filename (an os.PathLike, bytes, or str) to the filesystem
+        encoding with 'surrogateescape' error handler, return bytes unchanged.
+        On Windows, use 'strict' error handler if the file system encoding is
+        'mbcs' (which is the default encoding).
+        """
+        filename = fspath(filename)  # Does type-checking of `filename`.
+        if isinstance(filename, bytes):
+            return filename
+
+        try:
+            return filename.encode(encoding, errors)
+        except LookupError:
+            # Can't encode it, and the error handler doesn't
+            # exist. Probably on Python 2 with an astral character.
+            # Not sure how to handle this.
+            raise UnicodeEncodeError("Can't encode path to filesystem encoding")
+
+
+## Clocks
+try:
+    # Python 3.3+ (PEP 418)
+    from time import perf_counter
+    perf_counter = perf_counter
+except ImportError:
+    import time
+
+    if sys.platform == "win32":
+        perf_counter = time.clock
+    else:
+        perf_counter = time.time

python/gevent/_config.py

@@ -0,0 +1,709 @@
+# Copyright (c) 2018 gevent. See LICENSE for details.
+"""
+gevent tunables.
+
+This should be used as ``from gevent import config``. That variable
+is an object of :class:`Config`.
+
+.. versionadded:: 1.3a2
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import importlib
+import os
+import sys
+import textwrap
+
+from gevent._compat import string_types
+from gevent._compat import WIN
+
+__all__ = [
+    'config',
+]
+
+ALL_SETTINGS = []
+
+class SettingType(type):
+    # pylint:disable=bad-mcs-classmethod-argument
+
+    def __new__(cls, name, bases, cls_dict):
+        if name == 'Setting':
+            return type.__new__(cls, name, bases, cls_dict)
+
+        cls_dict["order"] = len(ALL_SETTINGS)
+        if 'name' not in cls_dict:
+            cls_dict['name'] = name.lower()
+
+        if 'environment_key' not in cls_dict:
+            cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()
+
+
+        new_class = type.__new__(cls, name, bases, cls_dict)
+        new_class.fmt_desc(cls_dict.get("desc", ""))
+        new_class.__doc__ = new_class.desc
+        ALL_SETTINGS.append(new_class)
+
+        if new_class.document:
+            setting_name = cls_dict['name']
+
+            def getter(self):
+                return self.settings[setting_name].get()
+
+            def setter(self, value): # pragma: no cover
+                # The setter should never be hit, Config has a
+                # __setattr__ that would override. But for the sake
+                # of consistency we provide one.
+                self.settings[setting_name].set(value)
+
+            prop = property(getter, setter, doc=new_class.__doc__)
+
+            setattr(Config, cls_dict['name'], prop)
+        return new_class
+
+    def fmt_desc(cls, desc):
+        desc = textwrap.dedent(desc).strip()
+        if hasattr(cls, 'shortname_map'):
+            desc += (
+                "\n\nThis is an importable value. It can be "
+                "given as a string naming an importable object, "
+                "or a list of strings in preference order and the first "
+                "successfully importable object will be used. (Separate values "
+                "in the environment variable with commas.) "
+                "It can also be given as the callable object itself (in code). "
+            )
+            if cls.shortname_map:
+                desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
+        if getattr(cls.validate, '__doc__'):
+            desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
+        if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
+            default = "`%s`" % (cls.default,)
+        else:
+            default = "`%r`" % (cls.default,)
+        desc += "\n\nThe default value is %s" % (default,)
+        desc += ("\n\nThe environment variable ``%s`` "
+                 "can be used to control this." % (cls.environment_key,))
+        setattr(cls, "desc", desc)
+        return desc
+
+def validate_invalid(value):
+    raise ValueError("Not a valid value: %r" % (value,))
+
+def validate_bool(value):
+    """
+    This is a boolean value.
+
+    In the environment variable, it may be given as ``1``, ``true``,
+    ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
+    ``no`` for `False`.
+    """
+    if isinstance(value, string_types):
+        value = value.lower().strip()
+        if value in ('1', 'true', 'on', 'yes'):
+            value = True
+        elif value in ('0', 'false', 'off', 'no') or not value:
+            value = False
+        else:
+            raise ValueError("Invalid boolean string: %r" % (value,))
+    return bool(value)
+
+def validate_anything(value):
+    return value
+
+convert_str_value_as_is = validate_anything
+
+class Setting(object):
+    name = None
+    value = None
+    validate = staticmethod(validate_invalid)
+    default = None
+    environment_key = None
+    document = True
+
+    desc = """\
+
+    A long ReST description.
+
+    The first line should be a single sentence.
+
+    """
+
+    def _convert(self, value):
+        if isinstance(value, string_types):
+            return value.split(',')
+        return value
+
+    def _default(self):
+        result = os.environ.get(self.environment_key, self.default)
+        result = self._convert(result)
+        return result
+
+    def get(self):
+        # If we've been specifically set, return it
+        if 'value' in self.__dict__:
+            return self.value
+        # Otherwise, read from the environment and reify
+        # so we return consistent results.
+        self.value = self.validate(self._default())
+        return self.value
+
+    def set(self, val):
+        self.value = self.validate(self._convert(val))
+
+
+Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))
+
+def make_settings():
+    """
+    Return fresh instances of all classes defined in `ALL_SETTINGS`.
+    """
+    settings = {}
+    for setting_kind in ALL_SETTINGS:
+        setting = setting_kind()
+        assert setting.name not in settings
+        settings[setting.name] = setting
+    return settings
+
+
+class Config(object):
+    """
+    Global configuration for gevent.
+
+    There is one instance of this object at ``gevent.config``. If you
+    are going to make changes in code, instead of using the documented
+    environment variables, you need to make the changes before using
+    any parts of gevent that might need those settings. For example::
+
+        >>> from gevent import config
+        >>> config.fileobject = 'thread'
+
+        >>> from gevent import fileobject
+        >>> fileobject.FileObject.__name__
+        'FileObjectThread'
+
+    .. versionadded:: 1.3a2
+
+    """
+
+    def __init__(self):
+        self.settings = make_settings()
+
+    def __getattr__(self, name):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        return self.settings[name].get()
+
+    def __setattr__(self, name, value):
+        if name != "settings" and name in self.settings:
+            self.set(name, value)
+        else:
+            super(Config, self).__setattr__(name, value)
+
+    def set(self, name, value):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        self.settings[name].set(value)
+
+    def __dir__(self):
+        return list(self.settings)
+
+
+class ImportableSetting(object):
+
+    def _import(self, path, _NONE=object):
+        # pylint:disable=too-many-branches
+        if isinstance(path, list):
+            if not path:
+                raise ImportError('Cannot import from empty list: %r' % (path, ))
+
+            for item in path[:-1]:
+                try:
+                    return self._import(item)
+                except ImportError:
+                    pass
+
+            return self._import(path[-1])
+
+        if not isinstance(path, string_types):
+            return path
+
+        if '.' not in path:
+            raise ImportError("Cannot import %r. "
+                              "Required format: [path/][package.]module.class. "
+                              "Or choose from %r"
+                              % (path, list(self.shortname_map)))
+
+        if '/' in path:
+            # This is dangerous, subject to race conditions, and
+            # may not work properly for things like namespace packages
+            import warnings
+            warnings.warn("Absolute paths are deprecated and will be removed in 1.4."
+                          "Please put the package on sys.path first",
+                          DeprecationWarning)
+            package_path, path = path.rsplit('/', 1)
+            sys.path = [package_path] + sys.path
+        else:
+            package_path = None
+
+        try:
+            module, item = path.rsplit('.', 1)
+            module = importlib.import_module(module)
+            x = getattr(module, item, _NONE)
+            if x is _NONE:
+                raise ImportError('Cannot import %r from %r' % (item, module))
+            return x
+        finally:
+            if package_path:
+                try:
+                    sys.path.remove(package_path)
+                except ValueError: # pragma: no cover
+                    pass
+
+    shortname_map = {}
+
+    def validate(self, value):
+        if isinstance(value, type):
+            return value
+        return self._import([self.shortname_map.get(x, x) for x in value])
+
+class BoolSettingMixin(object):
+    validate = staticmethod(validate_bool)
+    # Don't do string-to-list conversion.
+    _convert = staticmethod(convert_str_value_as_is)
+
+class IntSettingMixin(object):
+    # Don't do string-to-list conversion.
+    def _convert(self, value):
+        if value:
+            return int(value)
+
+    validate = staticmethod(validate_anything)
+
+
+class _PositiveValueMixin(object):
+
+    def validate(self, value):
+        if value is not None and value <= 0:
+            raise ValueError("Must be positive")
+        return value
+
+
+class FloatSettingMixin(_PositiveValueMixin):
+    def _convert(self, value):
+        if value:
+            return float(value)
+
+
+class ByteCountSettingMixin(_PositiveValueMixin):
+
+    _MULTIPLES = {
+        # All keys must be the same size.
+        'kb': 1024,
+        'mb': 1024 * 1024,
+        'gb': 1024 * 1024 * 1024,
+    }
+
+    _SUFFIX_SIZE = 2
+
+    def _convert(self, value):
+        if not value or not isinstance(value, str):
+            return value
+        value = value.lower()
+        for s, m in self._MULTIPLES.items():
+            if value[-self._SUFFIX_SIZE:] == s:
+                return int(value[:-self._SUFFIX_SIZE]) * m
+        return int(value)
+
+
+class Resolver(ImportableSetting, Setting):
+
+    desc = """\
+    The callable that will be used to create
+    :attr:`gevent.hub.Hub.resolver`.
+
+    See :doc:`dns` for more information.
+    """
+
+    default = [
+        'thread',
+        'dnspython',
+        'ares',
+        'block',
+    ]
+
+    shortname_map = {
+        'ares': 'gevent.resolver.ares.Resolver',
+        'thread': 'gevent.resolver.thread.Resolver',
+        'block': 'gevent.resolver.blocking.Resolver',
+        'dnspython': 'gevent.resolver.dnspython.Resolver',
+    }
+
+
+
+class Threadpool(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of threadpool we use.
+    """
+
+    default = 'gevent.threadpool.ThreadPool'
+
+
+class Loop(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of the loop we use.
+
+    On Windows, this defaults to libuv, while on
+    other platforms it defaults to libev.
+
+    """
+
+    default = [
+        'libev-cext',
+        'libev-cffi',
+        'libuv-cffi',
+    ] if not WIN else [
+        'libuv-cffi',
+        'libev-cext',
+        'libev-cffi',
+    ]
+
+    shortname_map = {
+        'libev-cext': 'gevent.libev.corecext.loop',
+        'libev-cffi': 'gevent.libev.corecffi.loop',
+        'libuv-cffi': 'gevent.libuv.loop.loop',
+    }
+
+    shortname_map['libuv'] = shortname_map['libuv-cffi']
+
+
+class FormatContext(ImportableSetting, Setting):
+    name = 'format_context'
+
+    # using pprint.pformat can override custom __repr__ methods on dict/list
+    # subclasses, which can be a security concern
+    default = 'pprint.saferepr'
+
+
+class LibevBackend(Setting):
+    name = 'libev_backend'
+    environment_key = 'GEVENT_BACKEND'
+
+    desc = """\
+    The backend for libev, such as 'select'
+    """
+
+    default = None
+
+    validate = staticmethod(validate_anything)
+
+
+class FileObject(ImportableSetting, Setting):
+    desc = """\
+    The kind of ``FileObject`` we will use.
+
+    See :mod:`gevent.fileobject` for a detailed description.
+
+    """
+    environment_key = 'GEVENT_FILE'
+
+    default = [
+        'posix',
+        'thread',
+    ]
+
+    shortname_map = {
+        'thread': 'gevent._fileobjectcommon.FileObjectThread',
+        'posix': 'gevent._fileobjectposix.FileObjectPosix',
+        'block': 'gevent._fileobjectcommon.FileObjectBlock'
+    }
+
+
+class WatchChildren(BoolSettingMixin, Setting):
+    desc = """\
+    Should we *not* watch children with the event loop watchers?
+
+    This is an advanced setting.
+
+    See :mod:`gevent.os` for a detailed description.
+    """
+    name = 'disable_watch_children'
+    environment_key = 'GEVENT_NOWAITPID'
+    default = False
+
+
+class TraceMalloc(IntSettingMixin, Setting):
+    name = 'trace_malloc'
+    environment_key = 'PYTHONTRACEMALLOC'
+    default = False
+
+    desc = """\
+    Should FFI objects track their allocation?
+
+    This is only useful for low-level debugging.
+
+    On Python 3, this environment variable is built in to the
+    interpreter, and it may also be set with the ``-X
+    tracemalloc`` command line argument.
+
+    On Python 2, gevent interprets this argument and adds extra
+    tracking information for FFI objects.
+    """
+
+
+class TrackGreenletTree(BoolSettingMixin, Setting):
+    name = 'track_greenlet_tree'
+    environment_key = 'GEVENT_TRACK_GREENLET_TREE'
+    default = True
+
+    desc = """\
+    Should `Greenlet` objects track their spawning tree?
+
+    Setting this to a false value will make spawning `Greenlet`
+    objects and using `spawn_raw` faster, but the
+    ``spawning_greenlet``, ``spawn_tree_locals`` and ``spawning_stack``
+    will not be captured.
+
+    .. versionadded:: 1.3b1
+    """
+
+
+## Monitoring settings
+# All env keys should begin with GEVENT_MONITOR
+
+class MonitorThread(BoolSettingMixin, Setting):
+    name = 'monitor_thread'
+    environment_key = 'GEVENT_MONITOR_THREAD_ENABLE'
+    default = False
+
+    desc = """\
+    Should each hub start a native OS thread to monitor
+    for problems?
+
+    Such a thread will periodically check to see if the event loop
+    is blocked for longer than `max_blocking_time`, producing output on
+    the hub's exception stream (stderr by default) if it detects this condition.
+
+    If this setting is true, then this thread will be created
+    the first time the hub is switched to,
+    or you can call :meth:`gevent.hub.Hub.start_periodic_monitoring_thread` at any
+    time to create it (from the same thread that will run the hub). That function
+    will return an instance of :class:`gevent.events.IPeriodicMonitorThread`
+    to which you can add your own monitoring functions. That function
+    also emits an event of :class:`gevent.events.PeriodicMonitorThreadStartedEvent`.
+
+    .. seealso:: `max_blocking_time`
+
+    .. versionadded:: 1.3b1
+    """
+
+class MaxBlockingTime(FloatSettingMixin, Setting):
+    name = 'max_blocking_time'
+    # This environment key doesn't follow the convention because it's
+    # meant to match a key used by existing projects
+    environment_key = 'GEVENT_MAX_BLOCKING_TIME'
+    default = 0.1
+
+    desc = """\
+    If the `monitor_thread` is enabled, this is
+    approximately how long (in seconds)
+    the event loop will be allowed to block before a warning is issued.
+
+    This function depends on using `greenlet.settrace`, so installing
+    your own trace function after starting the monitoring thread will
+    cause this feature to misbehave unless you call the function
+    returned by `greenlet.settrace`. If you install a tracing function *before*
+    the monitoring thread is started, it will still be called.
+
+    .. note:: In the unlikely event of creating and using multiple different
+        gevent hubs in the same native thread in a short period of time,
+        especially without destroying the hubs, false positives may be reported.
+
+    .. versionadded:: 1.3b1
+    """
+
+class MonitorMemoryPeriod(FloatSettingMixin, Setting):
+    name = 'memory_monitor_period'
+
+    environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD'
+    default = 5
+
+    desc = """\
+    If `monitor_thread` is enabled, this is approximately how long
+    (in seconds) we will go between checking the processes memory usage.
+
+    Checking the memory usage is relatively expensive on some operating
+    systems, so this should not be too low. gevent will place a floor
+    value on it.
+    """
+
+class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting):
+    name = 'max_memory_usage'
+
+    environment_key = 'GEVENT_MONITOR_MEMORY_MAX'
+    default = None
+
+    desc = """\
+    If `monitor_thread` is enabled,
+    then if memory usage exceeds this amount (in bytes), events will
+    be emitted. See `gevent.events`. In the environment variable, you can use
+    a suffix of 'kb', 'mb' or 'gb' to specify the value in kilobytes, megabytes
+    or gigibytes.
+
+    There is no default value for this setting. If you wish to
+    cap memory usage, you must choose a value.
+    """
+
+# The ares settings are all interpreted by
+# gevent/resolver/ares.pyx, so we don't do
+# any validation here.
+
+class AresSettingMixin(object):
+
+    document = False
+
+    @property
+    def kwarg_name(self):
+        return self.name[5:]
+
+    validate = staticmethod(validate_anything)
+
+    _convert = staticmethod(convert_str_value_as_is)
+
+class AresFlags(AresSettingMixin, Setting):
+    name = 'ares_flags'
+    default = None
+    environment_key = 'GEVENTARES_FLAGS'
+
+class AresTimeout(AresSettingMixin, Setting):
+    document = True
+    name = 'ares_timeout'
+    default = None
+    environment_key = 'GEVENTARES_TIMEOUT'
+    desc = """\
+
+    .. deprecated:: 1.3a2
+       Prefer the :attr:`resolver_timeout` setting. If both are set,
+       the results are not defined.
+    """
+
+class AresTries(AresSettingMixin, Setting):
+    name = 'ares_tries'
+    default = None
+    environment_key = 'GEVENTARES_TRIES'
+
+class AresNdots(AresSettingMixin, Setting):
+    name = 'ares_ndots'
+    default = None
+    environment_key = 'GEVENTARES_NDOTS'
+
+class AresUDPPort(AresSettingMixin, Setting):
+    name = 'ares_udp_port'
+    default = None
+    environment_key = 'GEVENTARES_UDP_PORT'
+
+class AresTCPPort(AresSettingMixin, Setting):
+    name = 'ares_tcp_port'
+    default = None
+    environment_key = 'GEVENTARES_TCP_PORT'
+
+class AresServers(AresSettingMixin, Setting):
+    document = True
+    name = 'ares_servers'
+    default = None
+    environment_key = 'GEVENTARES_SERVERS'
+    desc = """\
+    A list of strings giving the IP addresses of nameservers for the ares resolver.
+
+    In the environment variable, these strings are separated by commas.
+
+    .. deprecated:: 1.3a2
+       Prefer the :attr:`resolver_nameservers` setting. If both are set,
+       the results are not defined.
+    """
+
+# Generic nameservers, works for dnspython and ares.
+class ResolverNameservers(AresSettingMixin, Setting):
+    document = True
+    name = 'resolver_nameservers'
+    default = None
+    environment_key = 'GEVENT_RESOLVER_NAMESERVERS'
+    desc = """\
+    A list of strings giving the IP addresses of nameservers for the (non-system) resolver.
+
+    In the environment variable, these strings are separated by commas.
+
+    .. rubric:: Resolver Behaviour
+
+    * blocking
+
+      Ignored
+
+    * Threaded
+
+      Ignored
+
+    * dnspython
+
+      If this setting is not given, the dnspython resolver will
+      load nameservers to use from ``/etc/resolv.conf``
+      or the Windows registry. This setting replaces any nameservers read
+      from those means. Note that the file and registry are still read
+      for other settings.
+
+      .. caution:: dnspython does not validate the members of the list.
+         An improper address (such as a hostname instead of IP) has
+         undefined results, including hanging the process.
+
+    * ares
+
+      Similar to dnspython, but with more platform and compile-time
+      options. ares validates that the members of the list are valid
+      addresses.
+    """
+
+    # Normal string-to-list rules. But still validate_anything.
+    _convert = Setting._convert
+
+    # TODO: In the future, support reading a resolv.conf file
+    # *other* than /etc/resolv.conf, and do that both on Windows
+    # and other platforms. Also offer the option to disable the system
+    # configuration entirely.
+
+    @property
+    def kwarg_name(self):
+        return 'servers'
+
+# Generic timeout, works for dnspython and ares
+class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting):
+    document = True
+    name = 'resolver_timeout'
+    environment_key = 'GEVENT_RESOLVER_TIMEOUT'
+    desc = """\
+    The total amount of time that the DNS resolver will spend making queries.
+
+    Only the ares and dnspython resolvers support this.
+
+    .. versionadded:: 1.3a2
+    """
+
+    @property
+    def kwarg_name(self):
+        return 'timeout'
+
+config = Config()
+
+# Go ahead and attempt to import the loop when this class is
+# instantiated. The hub won't work if the loop can't be found. This
+# can solve problems with the class being imported from multiple
+# threads at once, leading to one of the imports failing.
+# factories are themselves handled lazily. See #687.
+
+# Don't cache it though, in case the user re-configures through the
+# API.
+
+try:
+    Loop().get()
+except ImportError: # pragma: no cover
+    pass

python/gevent/_event.pxd

@@ -0,0 +1,77 @@
+cimport cython
+
+from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop
+from gevent.__hub_local cimport get_hub_noargs as get_hub
+
+cdef _None
+cdef reraise
+cdef dump_traceback
+cdef load_traceback
+
+
+cdef InvalidSwitchError
+cdef Timeout
+cdef bint _greenlet_imported
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+cdef void _init()
+
+cdef class _AbstractLinkable:
+   # We declare the __weakref__ here in the base (even though
+   # that's not really what we want) as a workaround for a Cython
+   # issue we see reliably on 3.7b4 and sometimes on 3.6. See
+   # https://github.com/cython/cython/issues/2270
+   cdef object __weakref__
+   cdef _notifier
+   cdef set _links
+   cdef readonly SwitchOutGreenletWithLoop hub
+
+   cpdef rawlink(self, callback)
+   cpdef bint ready(self)
+   cpdef unlink(self, callback)
+
+   cdef _check_and_notify(self)
+   @cython.locals(todo=set)
+   cpdef _notify_links(self)
+   cdef _wait_core(self, timeout, catch=*)
+   cdef _wait_return_value(self, waited, wait_success)
+   cdef _wait(self, timeout=*)
+
+
+cdef class Event(_AbstractLinkable):
+   cdef bint _flag
+
+cdef class AsyncResult(_AbstractLinkable):
+    cdef readonly _value
+    cdef readonly tuple _exc_info
+
+    # For the use of _imap.py
+    cdef public int _imap_task_index
+
+    cpdef get(self, block=*, timeout=*)
+    cpdef bint successful(self)
+
+    cpdef wait(self, timeout=*)
+    cpdef bint done(self)
+
+    cpdef bint cancel(self)
+    cpdef bint cancelled(self)

python/gevent/_ffi/__init__.py

@@ -0,0 +1,27 @@
+"""
+Internal helpers for FFI implementations.
+"""
+from __future__ import print_function, absolute_import
+
+import os
+import sys
+
+def _dbg(*args, **kwargs):
+    # pylint:disable=unused-argument
+    pass
+
+#_dbg = print
+
+def _pid_dbg(*args, **kwargs):
+    kwargs['file'] = sys.stderr
+    print(os.getpid(), *args, **kwargs)
+
+CRITICAL = 1
+ERROR = 3
+DEBUG = 5
+TRACE = 9
+
+GEVENT_DEBUG_LEVEL = vars()[os.getenv("GEVENT_DEBUG", 'CRITICAL').upper()]
+
+if GEVENT_DEBUG_LEVEL >= TRACE:
+    _dbg = _pid_dbg

python/gevent/_ffi/callback.py

@@ -0,0 +1,58 @@
+from __future__ import absolute_import, print_function
+
+__all__ = [
+    'callback',
+]
+
+
+# For times when *args is captured but often not passed (empty),
+# we can avoid keeping the new tuple that was created for *args
+# around by using a constant.
+_NOARGS = ()
+
+
+class callback(object):
+
+    __slots__ = ('callback', 'args')
+
+    def __init__(self, cb, args):
+        self.callback = cb
+        self.args = args or _NOARGS
+
+    def stop(self):
+        self.callback = None
+        self.args = None
+
+    close = stop
+
+    # Note that __nonzero__ and pending are different
+    # bool() is used in contexts where we need to know whether to schedule another callback,
+    # so it's true if it's pending or currently running
+    # 'pending' has the same meaning as libev watchers: it is cleared before actually
+    # running the callback
+
+    def __nonzero__(self):
+        # it's nonzero if it's pending or currently executing
+        # NOTE: This depends on loop._run_callbacks setting the args property
+        # to None.
+        return self.args is not None
+    __bool__ = __nonzero__
+
+    @property
+    def pending(self):
+        return self.callback is not None
+
+    def _format(self):
+        return ''
+
+    def __repr__(self):
+        result = "<%s at 0x%x" % (self.__class__.__name__, id(self))
+        if self.pending:
+            result += " pending"
+        if self.callback is not None:
+            result += " callback=%r" % (self.callback, )
+        if self.args is not None:
+            result += " args=%r" % (self.args, )
+        if self.callback is None and self.args is None:
+            result += " stopped"
+        return result + ">"

python/gevent/_ffi/loop.py

@@ -0,0 +1,709 @@
+"""
+Basic loop implementation for ffi-based cores.
+"""
+# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable
+from __future__ import absolute_import, print_function
+
+from collections import deque
+import sys
+import os
+import traceback
+
+from gevent._ffi import _dbg
+from gevent._ffi import GEVENT_DEBUG_LEVEL
+from gevent._ffi import TRACE
+from gevent._ffi.callback import callback
+from gevent._compat import PYPY
+
+from gevent import getswitchinterval
+
+__all__ = [
+    'AbstractLoop',
+    'assign_standard_callbacks',
+]
+
+
+class _EVENTSType(object):
+    def __repr__(self):
+        return 'gevent.core.EVENTS'
+
+EVENTS = GEVENT_CORE_EVENTS = _EVENTSType()
+
+
+#####
+## Note on CFFI objects, callbacks and the lifecycle of watcher objects
+#
+# Each subclass of `watcher` allocates a C structure of the
+# appropriate type e.g., struct gevent_ev_io and holds this pointer in
+# its `_gwatcher` attribute. When that watcher instance is garbage
+# collected, then the C structure is also freed. The C structure is
+# passed to libev from the watcher's start() method and then to the
+# appropriate C callback function, e.g., _gevent_ev_io_callback, which
+# passes it back to python's _python_callback where we need the
+# watcher instance. Therefore, as long as that callback is active (the
+# watcher is started), the watcher instance must not be allowed to get
+# GC'd---any access at the C level or even the FFI level to the freed
+# memory could crash the process.
+#
+# However, the typical idiom calls for writing something like this:
+#  loop.io(fd, python_cb).start()
+# thus forgetting the newly created watcher subclass and allowing it to be immediately
+# GC'd. To combat this, when the watcher is started, it places itself into the loop's
+# `_keepaliveset`, and it only removes itself when the watcher's `stop()` method is called.
+# Often, this is the *only* reference keeping the watcher object, and hence its C structure,
+# alive.
+#
+# This is slightly complicated by the fact that the python-level
+# callback, called from the C callback, could choose to manually stop
+# the watcher. When we return to the C level callback, we now have an
+# invalid pointer, and attempting to pass it back to Python (e.g., to
+# handle an error) could crash. Hence, _python_callback,
+# _gevent_io_callback, and _python_handle_error cooperate to make sure
+# that the watcher instance stays in the loops `_keepaliveset` while
+# the C code could be running---and if it gets removed, to not call back
+# to Python again.
+# See also https://github.com/gevent/gevent/issues/676
+####
+class AbstractCallbacks(object):
+
+
+    def __init__(self, ffi):
+        self.ffi = ffi
+        self.callbacks = []
+        if GEVENT_DEBUG_LEVEL < TRACE:
+            self.from_handle = ffi.from_handle
+
+    def from_handle(self, handle): # pylint:disable=method-hidden
+        x = self.ffi.from_handle(handle)
+        return x
+
+    def python_callback(self, handle, revents):
+        """
+        Returns an integer having one of three values:
+
+        - -1
+          An exception occurred during the callback and you must call
+          :func:`_python_handle_error` to deal with it. The Python watcher
+          object will have the exception tuple saved in ``_exc_info``.
+        - 1
+          Everything went according to plan. You should check to see if the libev
+          watcher is still active, and call :func:`python_stop` if it is not. This will
+          clean up the memory. Finding the watcher still active at the event loop level,
+          but not having stopped itself at the gevent level is a buggy scenario and
+          shouldn't happen.
+        - 2
+          Everything went according to plan, but the watcher has already
+          been stopped. Its memory may no longer be valid.
+
+        This function should never return 0, as that's the default value that
+        Python exceptions will produce.
+        """
+        #print("Running callback", handle)
+        orig_ffi_watcher = None
+        try:
+            # Even dereferencing the handle needs to be inside the try/except;
+            # if we don't return normally (e.g., a signal) then we wind up going
+            # to the 'onerror' handler (unhandled_onerror), which
+            # is not what we want; that can permanently wedge the loop depending
+            # on which callback was executing.
+            # XXX: See comments in that function. We may be able to restart and do better?
+            if not handle:
+                # Hmm, a NULL handle. That's not supposed to happen.
+                # We can easily get into a loop if we deref it and allow that
+                # to raise.
+                _dbg("python_callback got null handle")
+                return 1
+            the_watcher = self.from_handle(handle)
+            orig_ffi_watcher = the_watcher._watcher
+            args = the_watcher.args
+            if args is None:
+                # Legacy behaviour from corecext: convert None into ()
+                # See test__core_watcher.py
+                args = _NOARGS
+            if args and args[0] == GEVENT_CORE_EVENTS:
+                args = (revents, ) + args[1:]
+            #print("Calling function", the_watcher.callback, args)
+            the_watcher.callback(*args)
+        except: # pylint:disable=bare-except
+            _dbg("Got exception servicing watcher with handle", handle, sys.exc_info())
+            # It's possible for ``the_watcher`` to be undefined (UnboundLocalError)
+            # if we threw an exception (signal) on the line that created that variable.
+            # This is typically the case with a signal under libuv
+            try:
+                the_watcher
+            except UnboundLocalError:
+                the_watcher = self.from_handle(handle)
+            the_watcher._exc_info = sys.exc_info()
+            # Depending on when the exception happened, the watcher
+            # may or may not have been stopped. We need to make sure its
+            # memory stays valid so we can stop it at the ev level if needed.
+            the_watcher.loop._keepaliveset.add(the_watcher)
+            return -1
+        else:
+            if (the_watcher.loop is not None
+                    and the_watcher in the_watcher.loop._keepaliveset
+                    and the_watcher._watcher is orig_ffi_watcher):
+                # It didn't stop itself, *and* it didn't stop itself, reset
+                # its watcher, and start itself again. libuv's io watchers MAY
+                # do that.
+                # The normal, expected scenario when we find the watcher still
+                # in the keepaliveset is that it is still active at the event loop
+                # level, so we don't expect that python_stop gets called.
+                #_dbg("The watcher has not stopped itself, possibly still active", the_watcher)
+                return 1
+            return 2 # it stopped itself
+
+    def python_handle_error(self, handle, _revents):
+        _dbg("Handling error for handle", handle)
+        if not handle:
+            return
+        try:
+            watcher = self.from_handle(handle)
+            exc_info = watcher._exc_info
+            del watcher._exc_info
+            # In the past, we passed the ``watcher`` itself as the context,
+            # which typically meant that the Hub would just print
+            # the exception. This is a problem because sometimes we can't
+            # detect signals until late in ``python_callback``; specifically,
+            # test_selectors.py:DefaultSelectorTest.test_select_interrupt_exc
+            # installs a SIGALRM handler that raises an exception. That exception can happen
+            # before we enter ``python_callback`` or at any point within it because of the way
+            # libuv swallows signals. By passing None, we get the exception prapagated into
+            # the main greenlet (which is probably *also* not what we always want, but
+            # I see no way to distinguish the cases).
+            watcher.loop.handle_error(None, *exc_info)
+        finally:
+            # XXX Since we're here on an error condition, and we
+            # made sure that the watcher object was put in loop._keepaliveset,
+            # what about not stopping the watcher? Looks like a possible
+            # memory leak?
+            # XXX: This used to do "if revents & (libev.EV_READ | libev.EV_WRITE)"
+            # before stopping. Why?
+            try:
+                watcher.stop()
+            except: # pylint:disable=bare-except
+                watcher.loop.handle_error(watcher, *sys.exc_info())
+            return # pylint:disable=lost-exception
+
+    def unhandled_onerror(self, t, v, tb):
+        # This is supposed to be called for signals, etc.
+        # This is the onerror= value for CFFI.
+        # If we return None, C will get a value of 0/NULL;
+        # if we raise, CFFI will print the exception and then
+        # return 0/NULL; (unless error= was configured)
+        # If things go as planned, we return the value that asks
+        # C to call back and check on if the watcher needs to be closed or
+        # not.
+
+        # XXX: TODO: Could this cause events to be lost? Maybe we need to return
+        # a value that causes the C loop to try the callback again?
+        # at least for signals under libuv, which are delivered at very odd times.
+        # Hopefully the event still shows up when we poll the next time.
+        watcher = None
+        handle = tb.tb_frame.f_locals['handle'] if tb is not None else None
+        if handle: # handle could be NULL
+            watcher = self.from_handle(handle)
+        if watcher is not None:
+            watcher.loop.handle_error(None, t, v, tb)
+            return 1
+
+        # Raising it causes a lot of noise from CFFI
+        print("WARNING: gevent: Unhandled error with no watcher",
+              file=sys.stderr)
+        traceback.print_exception(t, v, tb)
+
+    def python_stop(self, handle):
+        if not handle: # pragma: no cover
+            print(
+                "WARNING: gevent: Unable to dereference handle; not stopping watcher. "
+                "Native resources may leak. This is most likely a bug in gevent.",
+                file=sys.stderr)
+            # The alternative is to crash with no helpful information
+            # NOTE: Raising exceptions here does nothing, they're swallowed by CFFI.
+            # Since the C level passed in a null pointer, even dereferencing the handle
+            # will just produce some exceptions.
+            return
+        watcher = self.from_handle(handle)
+        watcher.stop()
+
+    if not PYPY:
+        def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument
+            # If we have the onerror callback, this is a no-op; all the real
+            # work to rethrow the exception is done by the onerror callback
+
+            # NOTE: Unlike the rest of the functions, this is called with a pointer
+            # to the C level structure, *not* a pointer to the void* that represents a
+            # <cdata> for the Python Watcher object.
+            pass
+    else: # PyPy
+        # On PyPy, we need the function to have some sort of body, otherwise
+        # the signal exceptions don't always get caught, *especially* with
+        # libuv (however, there's no reason to expect this to only be a libuv
+        # issue; it's just that we don't depend on the periodic signal timer
+        # under libev, so the issue is much more pronounced under libuv)
+        # test_socket's test_sendall_interrupted can hang.
+        # See https://github.com/gevent/gevent/issues/1112
+
+        def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument
+            # Things we've tried that *don't* work:
+            # greenlet.getcurrent()
+            # 1 + 1
+            try:
+                raise MemoryError()
+            except MemoryError:
+                pass
+
+    def python_prepare_callback(self, watcher_ptr):
+        loop = self._find_loop_from_c_watcher(watcher_ptr)
+        if loop is None: # pragma: no cover
+            print("WARNING: gevent: running prepare callbacks from a destroyed handle: ",
+                  watcher_ptr)
+            return
+        loop._run_callbacks()
+
+    def check_callback_onerror(self, t, v, tb):
+        watcher_ptr = tb.tb_frame.f_locals['watcher_ptr'] if tb is not None else None
+        if watcher_ptr:
+            loop = self._find_loop_from_c_watcher(watcher_ptr)
+        if loop is not None:
+            # None as the context argument causes the exception to be raised
+            # in the main greenlet.
+            loop.handle_error(None, t, v, tb)
+            return None
+        raise v # Let CFFI print
+
+    def _find_loop_from_c_watcher(self, watcher_ptr):
+        raise NotImplementedError()
+
+
+
+def assign_standard_callbacks(ffi, lib, callbacks_class, extras=()): # pylint:disable=unused-argument
+    # callbacks keeps these cdata objects alive at the python level
+    callbacks = callbacks_class(ffi)
+    extras = tuple([(getattr(callbacks, name), error) for name, error in extras])
+    for (func, error_func) in ((callbacks.python_callback, None),
+                               (callbacks.python_handle_error, None),
+                               (callbacks.python_stop, None),
+                               (callbacks.python_check_callback,
+                                callbacks.check_callback_onerror),
+                               (callbacks.python_prepare_callback,
+                                callbacks.check_callback_onerror)) + extras:
+        # The name of the callback function matches the 'extern Python' declaration.
+        error_func = error_func or callbacks.unhandled_onerror
+        callback = ffi.def_extern(onerror=error_func)(func)
+        # keep alive the cdata
+        # (def_extern returns the original function, and it requests that
+        # the function be "global", so maybe it keeps a hard reference to it somewhere now
+        # unlike ffi.callback(), and we don't need to do this?)
+        callbacks.callbacks.append(callback)
+
+        # At this point, the library C variable (static function, actually)
+        # is filled in.
+
+    return callbacks
+
+
+if sys.version_info[0] >= 3:
+    basestring = (bytes, str)
+    integer_types = (int,)
+else:
+    import __builtin__ # pylint:disable=import-error
+    basestring = (__builtin__.basestring,)
+    integer_types = (int, __builtin__.long)
+
+
+
+
+_NOARGS = ()
+
+CALLBACK_CHECK_COUNT = 50
+
+class AbstractLoop(object):
+    # pylint:disable=too-many-public-methods,too-many-instance-attributes
+
+    error_handler = None
+
+    _CHECK_POINTER = None
+
+    _TIMER_POINTER = None
+    _TIMER_CALLBACK_SIG = None
+
+    _PREPARE_POINTER = None
+
+    starting_timer_may_update_loop_time = False
+
+    # Subclasses should set this in __init__ to reflect
+    # whether they were the default loop.
+    _default = None
+
+    def __init__(self, ffi, lib, watchers, flags=None, default=None):
+        self._ffi = ffi
+        self._lib = lib
+        self._ptr = None
+        self._handle_to_self = self._ffi.new_handle(self) # XXX: Reference cycle?
+        self._watchers = watchers
+        self._in_callback = False
+        self._callbacks = deque()
+        # Stores python watcher objects while they are started
+        self._keepaliveset = set()
+        self._init_loop_and_aux_watchers(flags, default)
+
+
+    def _init_loop_and_aux_watchers(self, flags=None, default=None):
+
+        self._ptr = self._init_loop(flags, default)
+
+
+        # self._check is a watcher that runs in each iteration of the
+        # mainloop, just after the blocking call. It's point is to handle
+        # signals. It doesn't run watchers or callbacks, it just exists to give
+        # CFFI a chance to raise signal exceptions so we can handle them.
+        self._check = self._ffi.new(self._CHECK_POINTER)
+        self._check.data = self._handle_to_self
+        self._init_and_start_check()
+
+        # self._prepare is a watcher that runs in each iteration of the mainloop,
+        # just before the blocking call. It's where we run deferred callbacks
+        # from self.run_callback. This cooperates with _setup_for_run_callback()
+        # to schedule self._timer0 if needed.
+        self._prepare = self._ffi.new(self._PREPARE_POINTER)
+        self._prepare.data = self._handle_to_self
+        self._init_and_start_prepare()
+
+        # A timer we start and stop on demand. If we have callbacks,
+        # too many to run in one iteration of _run_callbacks, we turn this
+        # on so as to have the next iteration of the run loop return to us
+        # as quickly as possible.
+        # TODO: There may be a more efficient way to do this using ev_timer_again;
+        # see the "ev_timer" section of the ev manpage (http://linux.die.net/man/3/ev)
+        # Alternatively, setting the ev maximum block time may also work.
+        self._timer0 = self._ffi.new(self._TIMER_POINTER)
+        self._timer0.data = self._handle_to_self
+        self._init_callback_timer()
+
+        # TODO: We may be able to do something nicer and use the existing python_callback
+        # combined with onerror and the class check/timer/prepare to simplify things
+        # and unify our handling
+
+    def _init_loop(self, flags, default):
+        """
+        Called by __init__ to create or find the loop. The return value
+        is assigned to self._ptr.
+        """
+        raise NotImplementedError()
+
+    def _init_and_start_check(self):
+        raise NotImplementedError()
+
+    def _init_and_start_prepare(self):
+        raise NotImplementedError()
+
+    def _init_callback_timer(self):
+        raise NotImplementedError()
+
+    def _stop_callback_timer(self):
+        raise NotImplementedError()
+
+    def _start_callback_timer(self):
+        raise NotImplementedError()
+
+    def _check_callback_handle_error(self, t, v, tb):
+        self.handle_error(None, t, v, tb)
+
+    def _run_callbacks(self): # pylint:disable=too-many-branches
+        # When we're running callbacks, its safe for timers to
+        # update the notion of the current time (because if we're here,
+        # we're not running in a timer callback that may let other timers
+        # run; this is mostly an issue for libuv).
+
+        # That's actually a bit of a lie: on libev, self._timer0 really is
+        # a timer, and so sometimes this is running in a timer callback, not
+        # a prepare callback. But that's OK, libev doesn't suffer from cascading
+        # timer expiration and its safe to update the loop time at any
+        # moment there.
+        self.starting_timer_may_update_loop_time = True
+        try:
+            count = CALLBACK_CHECK_COUNT
+            now = self.now()
+            expiration = now + getswitchinterval()
+            self._stop_callback_timer()
+            while self._callbacks:
+                cb = self._callbacks.popleft() # pylint:disable=assignment-from-no-return
+                count -= 1
+                self.unref() # XXX: libuv doesn't have a global ref count!
+                callback = cb.callback
+                cb.callback = None
+                args = cb.args
+                if callback is None or args is None:
+                    # it's been stopped
+                    continue
+
+                try:
+                    callback(*args)
+                except: # pylint:disable=bare-except
+                    # If we allow an exception to escape this method (while we are running the ev callback),
+                    # then CFFI will print the error and libev will continue executing.
+                    # There are two problems with this. The first is that the code after
+                    # the loop won't run. The second is that any remaining callbacks scheduled
+                    # for this loop iteration will be silently dropped; they won't run, but they'll
+                    # also not be *stopped* (which is not a huge deal unless you're looking for
+                    # consistency or checking the boolean/pending status; the loop doesn't keep
+                    # a reference to them like it does to watchers...*UNLESS* the callback itself had
+                    # a reference to a watcher; then I don't know what would happen, it depends on
+                    # the state of the watcher---a leak or crash is not totally inconceivable).
+                    # The Cython implementation in core.ppyx uses gevent_call from callbacks.c
+                    # to run the callback, which uses gevent_handle_error to handle any errors the
+                    # Python callback raises...it unconditionally simply prints any error raised
+                    # by loop.handle_error and clears it, so callback handling continues.
+                    # We take a similar approach (but are extra careful about printing)
+                    try:
+                        self.handle_error(cb, *sys.exc_info())
+                    except: # pylint:disable=bare-except
+                        try:
+                            print("Exception while handling another error", file=sys.stderr)
+                            traceback.print_exc()
+                        except: # pylint:disable=bare-except
+                            pass # Nothing we can do here
+                finally:
+                    # NOTE: this must be reset here, because cb.args is used as a flag in
+                    # the callback class so that bool(cb) of a callback that has been run
+                    # becomes False
+                    cb.args = None
+
+                # We've finished running one group of callbacks
+                # but we may have more, so before looping check our
+                # switch interval.
+                if count == 0 and self._callbacks:
+                    count = CALLBACK_CHECK_COUNT
+                    self.update_now()
+                    if self.now() >= expiration:
+                        now = 0
+                        break
+
+            # Update the time before we start going again, if we didn't
+            # just do so.
+            if now != 0:
+                self.update_now()
+
+            if self._callbacks:
+                self._start_callback_timer()
+        finally:
+            self.starting_timer_may_update_loop_time = False
+
+    def _stop_aux_watchers(self):
+        raise NotImplementedError()
+
+    def destroy(self):
+        if self._ptr:
+            try:
+                if not self._can_destroy_loop(self._ptr):
+                    return False
+                self._stop_aux_watchers()
+                self._destroy_loop(self._ptr)
+            finally:
+                # not ffi.NULL, we don't want something that can be
+                # passed to C and crash later. This will create nice friendly
+                # TypeError from CFFI.
+                self._ptr = None
+                del self._handle_to_self
+                del self._callbacks
+                del self._keepaliveset
+
+            return True
+
+    def _can_destroy_loop(self, ptr):
+        raise NotImplementedError()
+
+    def _destroy_loop(self, ptr):
+        raise NotImplementedError()
+
+    @property
+    def ptr(self):
+        return self._ptr
+
+    @property
+    def WatcherType(self):
+        return self._watchers.watcher
+
+    @property
+    def MAXPRI(self):
+        return 1
+
+    @property
+    def MINPRI(self):
+        return 1
+
+    def _handle_syserr(self, message, errno):
+        try:
+            errno = os.strerror(errno)
+        except: # pylint:disable=bare-except
+            traceback.print_exc()
+        try:
+            message = '%s: %s' % (message, errno)
+        except: # pylint:disable=bare-except
+            traceback.print_exc()
+        self.handle_error(None, SystemError, SystemError(message), None)
+
+    def handle_error(self, context, type, value, tb):
+        handle_error = None
+        error_handler = self.error_handler
+        if error_handler is not None:
+            # we do want to do getattr every time so that setting Hub.handle_error property just works
+            handle_error = getattr(error_handler, 'handle_error', error_handler)
+            handle_error(context, type, value, tb)
+        else:
+            self._default_handle_error(context, type, value, tb)
+
+    def _default_handle_error(self, context, type, value, tb): # pylint:disable=unused-argument
+        # note: Hub sets its own error handler so this is not used by gevent
+        # this is here to make core.loop usable without the rest of gevent
+        # Should cause the loop to stop running.
+        traceback.print_exception(type, value, tb)
+
+
+    def run(self, nowait=False, once=False):
+        raise NotImplementedError()
+
+    def reinit(self):
+        raise NotImplementedError()
+
+    def ref(self):
+        # XXX: libuv doesn't do it this way
+        raise NotImplementedError()
+
+    def unref(self):
+        raise NotImplementedError()
+
+    def break_(self, how=None):
+        raise NotImplementedError()
+
+    def verify(self):
+        pass
+
+    def now(self):
+        raise NotImplementedError()
+
+    def update_now(self):
+        raise NotImplementedError()
+
+    def update(self):
+        import warnings
+        warnings.warn("'update' is deprecated; use 'update_now'",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self.update_now()
+
+    def __repr__(self):
+        return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self._format())
+
+    @property
+    def default(self):
+        return self._default if self._ptr else False
+
+    @property
+    def iteration(self):
+        return -1
+
+    @property
+    def depth(self):
+        return -1
+
+    @property
+    def backend_int(self):
+        return 0
+
+    @property
+    def backend(self):
+        return "default"
+
+    @property
+    def pendingcnt(self):
+        return 0
+
+    def io(self, fd, events, ref=True, priority=None):
+        return self._watchers.io(self, fd, events, ref, priority)
+
+    def timer(self, after, repeat=0.0, ref=True, priority=None):
+        return self._watchers.timer(self, after, repeat, ref, priority)
+
+    def signal(self, signum, ref=True, priority=None):
+        return self._watchers.signal(self, signum, ref, priority)
+
+    def idle(self, ref=True, priority=None):
+        return self._watchers.idle(self, ref, priority)
+
+    def prepare(self, ref=True, priority=None):
+        return self._watchers.prepare(self, ref, priority)
+
+    def check(self, ref=True, priority=None):
+        return self._watchers.check(self, ref, priority)
+
+    def fork(self, ref=True, priority=None):
+        return self._watchers.fork(self, ref, priority)
+
+    def async_(self, ref=True, priority=None):
+        return self._watchers.async_(self, ref, priority)
+
+    # Provide BWC for those that can use 'async' as is
+    locals()['async'] = async_
+
+    if sys.platform != "win32":
+
+        def child(self, pid, trace=0, ref=True):
+            return self._watchers.child(self, pid, trace, ref)
+
+        def install_sigchld(self):
+            pass
+
+    def stat(self, path, interval=0.0, ref=True, priority=None):
+        return self._watchers.stat(self, path, interval, ref, priority)
+
+    def callback(self, priority=None):
+        return callback(self, priority)
+
+    def _setup_for_run_callback(self):
+        raise NotImplementedError()
+
+    def run_callback(self, func, *args):
+        # If we happen to already be running callbacks (inside
+        # _run_callbacks), this could happen almost immediately,
+        # without the loop cycling.
+        cb = callback(func, args)
+        self._callbacks.append(cb)
+        self._setup_for_run_callback()
+
+        return cb
+
+    def _format(self):
+        if not self._ptr:
+            return 'destroyed'
+        msg = self.backend
+        if self.default:
+            msg += ' default'
+        msg += ' pending=%s' % self.pendingcnt
+        msg += self._format_details()
+        return msg
+
+    def _format_details(self):
+        msg = ''
+        fileno = self.fileno() # pylint:disable=assignment-from-none
+        try:
+            activecnt = self.activecnt
+        except AttributeError:
+            activecnt = None
+        if activecnt is not None:
+            msg += ' ref=' + repr(activecnt)
+        if fileno is not None:
+            msg += ' fileno=' + repr(fileno)
+        #if sigfd is not None and sigfd != -1:
+        #    msg += ' sigfd=' + repr(sigfd)
+        return msg
+
+    def fileno(self):
+        return None
+
+    @property
+    def activecnt(self):
+        if not self._ptr:
+            raise ValueError('operation on destroyed loop')
+        return 0

python/gevent/_ffi/watcher.py

@@ -0,0 +1,641 @@
+"""
+Useful base classes for watchers. The available
+watchers will depend on the specific event loop.
+"""
+# pylint:disable=not-callable
+from __future__ import absolute_import, print_function
+
+import signal as signalmodule
+import functools
+import warnings
+
+from gevent._config import config
+
+try:
+    from tracemalloc import get_object_traceback
+
+    def tracemalloc(init):
+        # PYTHONTRACEMALLOC env var controls this on Python 3.
+        return init
+except ImportError: # Python < 3.4
+
+    if config.trace_malloc:
+        # Use the same env var to turn this on for Python 2
+        import traceback
+
+        class _TB(object):
+            __slots__ = ('lines',)
+
+            def __init__(self, lines):
+                # These end in newlines, which we don't want for consistency
+                self.lines = [x.rstrip() for x in lines]
+
+            def format(self):
+                return self.lines
+
+        def tracemalloc(init):
+            @functools.wraps(init)
+            def traces(self, *args, **kwargs):
+                init(self, *args, **kwargs)
+                self._captured_malloc = _TB(traceback.format_stack())
+            return traces
+
+        def get_object_traceback(obj):
+            return obj._captured_malloc
+
+    else:
+        def get_object_traceback(_obj):
+            return None
+
+        def tracemalloc(init):
+            return init
+
+from gevent._compat import fsencode
+
+from gevent._ffi import _dbg # pylint:disable=unused-import
+from gevent._ffi import GEVENT_DEBUG_LEVEL
+from gevent._ffi import DEBUG
+from gevent._ffi.loop import GEVENT_CORE_EVENTS
+from gevent._ffi.loop import _NOARGS
+
+ALLOW_WATCHER_DEL = GEVENT_DEBUG_LEVEL >= DEBUG
+
+__all__ = [
+
+]
+
+try:
+    ResourceWarning
+except NameError:
+    class ResourceWarning(Warning):
+        "Python 2 fallback"
+
+class _NoWatcherResult(int):
+
+    def __repr__(self):
+        return "<NoWatcher>"
+
+_NoWatcherResult = _NoWatcherResult(0)
+
+def events_to_str(event_field, all_events):
+    result = []
+    for (flag, string) in all_events:
+        c_flag = flag
+        if event_field & c_flag:
+            result.append(string)
+            event_field = event_field & (~c_flag)
+        if not event_field:
+            break
+    if event_field:
+        result.append(hex(event_field))
+    return '|'.join(result)
+
+
+def not_while_active(func):
+    @functools.wraps(func)
+    def nw(self, *args, **kwargs):
+        if self.active:
+            raise ValueError("not while active")
+        func(self, *args, **kwargs)
+    return nw
+
+def only_if_watcher(func):
+    @functools.wraps(func)
+    def if_w(self):
+        if self._watcher:
+            return func(self)
+        return _NoWatcherResult
+    return if_w
+
+
+class LazyOnClass(object):
+
+    @classmethod
+    def lazy(cls, cls_dict, func):
+        "Put a LazyOnClass object in *cls_dict* with the same name as *func*"
+        cls_dict[func.__name__] = cls(func)
+
+    def __init__(self, func, name=None):
+        self.name = name or func.__name__
+        self.func = func
+
+    def __get__(self, inst, klass):
+        if inst is None: # pragma: no cover
+            return self
+
+        val = self.func(inst)
+        setattr(klass, self.name, val)
+        return val
+
+
+class AbstractWatcherType(type):
+    """
+    Base metaclass for watchers.
+
+    To use, you will:
+
+    - subclass the watcher class defined from this type.
+    - optionally subclass this type
+    """
+    # pylint:disable=bad-mcs-classmethod-argument
+
+    _FFI = None
+    _LIB = None
+
+    def __new__(cls, name, bases, cls_dict):
+        if name != 'watcher' and not cls_dict.get('_watcher_skip_ffi'):
+            cls._fill_watcher(name, bases, cls_dict)
+        if '__del__' in cls_dict and not ALLOW_WATCHER_DEL: # pragma: no cover
+            raise TypeError("CFFI watchers are not allowed to have __del__")
+        return type.__new__(cls, name, bases, cls_dict)
+
+    @classmethod
+    def _fill_watcher(cls, name, bases, cls_dict):
+        # TODO: refactor smaller
+        # pylint:disable=too-many-locals
+        if name.endswith('_'):
+            # Strip trailing _ added to avoid keyword duplications
+            # e.g., async_
+            name = name[:-1]
+
+        def _mro_get(attr, bases, error=True):
+            for b in bases:
+                try:
+                    return getattr(b, attr)
+                except AttributeError:
+                    continue
+            if error: # pragma: no cover
+                raise AttributeError(attr)
+        _watcher_prefix = cls_dict.get('_watcher_prefix') or _mro_get('_watcher_prefix', bases)
+
+        if '_watcher_type' not in cls_dict:
+            watcher_type = _watcher_prefix + '_' + name
+            cls_dict['_watcher_type'] = watcher_type
+        elif not cls_dict['_watcher_type'].startswith(_watcher_prefix):
+            watcher_type = _watcher_prefix + '_' + cls_dict['_watcher_type']
+            cls_dict['_watcher_type'] = watcher_type
+
+        active_name = _watcher_prefix + '_is_active'
+
+        def _watcher_is_active(self):
+            return getattr(self._LIB, active_name)
+
+        LazyOnClass.lazy(cls_dict, _watcher_is_active)
+
+        watcher_struct_name = cls_dict.get('_watcher_struct_name')
+        if not watcher_struct_name:
+            watcher_struct_pattern = (cls_dict.get('_watcher_struct_pattern')
+                                      or _mro_get('_watcher_struct_pattern', bases, False)
+                                      or 'struct %s')
+            watcher_struct_name = watcher_struct_pattern % (watcher_type,)
+
+        def _watcher_struct_pointer_type(self):
+            return self._FFI.typeof(watcher_struct_name + ' *')
+
+        LazyOnClass.lazy(cls_dict, _watcher_struct_pointer_type)
+
+        callback_name = (cls_dict.get('_watcher_callback_name')
+                         or _mro_get('_watcher_callback_name', bases, False)
+                         or '_gevent_generic_callback')
+
+        def _watcher_callback(self):
+            return self._FFI.addressof(self._LIB, callback_name)
+
+        LazyOnClass.lazy(cls_dict, _watcher_callback)
+
+        def _make_meth(name, watcher_name):
+            def meth(self):
+                lib_name = self._watcher_type + '_' + name
+                return getattr(self._LIB, lib_name)
+            meth.__name__ = watcher_name
+            return meth
+
+        for meth_name in 'start', 'stop', 'init':
+            watcher_name = '_watcher' + '_' + meth_name
+            if watcher_name not in cls_dict:
+                LazyOnClass.lazy(cls_dict, _make_meth(meth_name, watcher_name))
+
+    def new_handle(cls, obj):
+        return cls._FFI.new_handle(obj)
+
+    def new(cls, kind):
+        return cls._FFI.new(kind)
+
+class watcher(object):
+
+    _callback = None
+    _args = None
+    _watcher = None
+    # self._handle has a reference to self, keeping it alive.
+    # We must keep self._handle alive for ffi.from_handle() to be
+    # able to work. We only fill this in when we are started,
+    # and when we are stopped we destroy it.
+    # NOTE: This is a GC cycle, so we keep it around for as short
+    # as possible.
+    _handle = None
+
+    @tracemalloc
+    def __init__(self, _loop, ref=True, priority=None, args=_NOARGS):
+        self.loop = _loop
+        self.__init_priority = priority
+        self.__init_args = args
+        self.__init_ref = ref
+        self._watcher_full_init()
+
+
+    def _watcher_full_init(self):
+        priority = self.__init_priority
+        ref = self.__init_ref
+        args = self.__init_args
+
+        self._watcher_create(ref)
+
+        if priority is not None:
+            self._watcher_ffi_set_priority(priority)
+
+        try:
+            self._watcher_ffi_init(args)
+        except:
+            # Let these be GC'd immediately.
+            # If we keep them around to when *we* are gc'd,
+            # they're probably invalid, meaning any native calls
+            # we do then to close() them are likely to fail
+            self._watcher = None
+            raise
+        self._watcher_ffi_set_init_ref(ref)
+
+    @classmethod
+    def _watcher_ffi_close(cls, ffi_watcher):
+        pass
+
+    def _watcher_create(self, ref): # pylint:disable=unused-argument
+        self._watcher = self._watcher_new()
+
+    def _watcher_new(self):
+        return type(self).new(self._watcher_struct_pointer_type) # pylint:disable=no-member
+
+    def _watcher_ffi_set_init_ref(self, ref):
+        pass
+
+    def _watcher_ffi_set_priority(self, priority):
+        pass
+
+    def _watcher_ffi_init(self, args):
+        raise NotImplementedError()
+
+    def _watcher_ffi_start(self):
+        raise NotImplementedError()
+
+    def _watcher_ffi_stop(self):
+        self._watcher_stop(self.loop._ptr, self._watcher)
+
+    def _watcher_ffi_ref(self):
+        raise NotImplementedError()
+
+    def _watcher_ffi_unref(self):
+        raise NotImplementedError()
+
+    def _watcher_ffi_start_unref(self):
+        # While a watcher is active, we don't keep it
+        # referenced. This allows a timer, for example, to be started,
+        # and still allow the loop to end if there is nothing
+        # else to do. see test__order.TestSleep0 for one example.
+        self._watcher_ffi_unref()
+
+    def _watcher_ffi_stop_ref(self):
+        self._watcher_ffi_ref()
+
+    # A string identifying the type of libev object we watch, e.g., 'ev_io'
+    # This should be a class attribute.
+    _watcher_type = None
+    # A class attribute that is the callback on the libev object that init's the C struct,
+    # e.g., libev.ev_io_init. If None, will be set by _init_subclasses.
+    _watcher_init = None
+    # A class attribute that is the callback on the libev object that starts the C watcher,
+    # e.g., libev.ev_io_start. If None, will be set by _init_subclasses.
+    _watcher_start = None
+    # A class attribute that is the callback on the libev object that stops the C watcher,
+    # e.g., libev.ev_io_stop. If None, will be set by _init_subclasses.
+    _watcher_stop = None
+    # A cffi ctype object identifying the struct pointer we create.
+    # This is a class attribute set based on the _watcher_type
+    _watcher_struct_pointer_type = None
+    # The attribute of the libev object identifying the custom
+    # callback function for this type of watcher. This is a class
+    # attribute set based on the _watcher_type in _init_subclasses.
+    _watcher_callback = None
+    _watcher_is_active = None
+
+    def close(self):
+        if self._watcher is None:
+            return
+
+        self.stop()
+        _watcher = self._watcher
+        self._watcher = None
+        self._watcher_set_data(_watcher, self._FFI.NULL) # pylint: disable=no-member
+        self._watcher_ffi_close(_watcher)
+        self.loop = None
+
+    def _watcher_set_data(self, the_watcher, data):
+        # This abstraction exists for the sole benefit of
+        # libuv.watcher.stat, which "subclasses" uv_handle_t.
+        # Can we do something to avoid this extra function call?
+        the_watcher.data = data
+        return data
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, t, v, tb):
+        self.close()
+
+    if ALLOW_WATCHER_DEL:
+        def __del__(self):
+            if self._watcher:
+                tb = get_object_traceback(self)
+                tb_msg = ''
+                if tb is not None:
+                    tb_msg = '\n'.join(tb.format())
+                    tb_msg = '\nTraceback:\n' + tb_msg
+                warnings.warn("Failed to close watcher %r%s" % (self, tb_msg),
+                              ResourceWarning)
+
+                # may fail if __init__ did; will be harmlessly printed
+                self.close()
+
+
+    def __repr__(self):
+        formats = self._format()
+        result = "<%s at 0x%x%s" % (self.__class__.__name__, id(self), formats)
+        if self.pending:
+            result += " pending"
+        if self.callback is not None:
+            fself = getattr(self.callback, '__self__', None)
+            if fself is self:
+                result += " callback=<bound method %s of self>" % (self.callback.__name__)
+            else:
+                result += " callback=%r" % (self.callback, )
+        if self.args is not None:
+            result += " args=%r" % (self.args, )
+        if self.callback is None and self.args is None:
+            result += " stopped"
+        result += " watcher=%s" % (self._watcher)
+        result += " handle=%s" % (self._watcher_handle)
+        result += " ref=%s" % (self.ref)
+        return result + ">"
+
+    @property
+    def _watcher_handle(self):
+        if self._watcher:
+            return self._watcher.data
+
+    def _format(self):
+        return ''
+
+    @property
+    def ref(self):
+        raise NotImplementedError()
+
+    def _get_callback(self):
+        return self._callback
+
+    def _set_callback(self, cb):
+        if not callable(cb) and cb is not None:
+            raise TypeError("Expected callable, not %r" % (cb, ))
+        if cb is None:
+            if '_callback' in self.__dict__:
+                del self._callback
+        else:
+            self._callback = cb
+    callback = property(_get_callback, _set_callback)
+
+    def _get_args(self):
+        return self._args
+
+    def _set_args(self, args):
+        if not isinstance(args, tuple) and args is not None:
+            raise TypeError("args must be a tuple or None")
+        if args is None:
+            if '_args' in self.__dict__:
+                del self._args
+        else:
+            self._args = args
+
+    args = property(_get_args, _set_args)
+
+    def start(self, callback, *args):
+        if callback is None:
+            raise TypeError('callback must be callable, not None')
+        self.callback = callback
+        self.args = args or _NOARGS
+        self.loop._keepaliveset.add(self)
+        self._handle = self._watcher_set_data(self._watcher, type(self).new_handle(self)) # pylint:disable=no-member
+        self._watcher_ffi_start()
+        self._watcher_ffi_start_unref()
+
+    def stop(self):
+        if self._callback is None:
+            assert self.loop is None or self not in self.loop._keepaliveset
+            return
+        self._watcher_ffi_stop_ref()
+        self._watcher_ffi_stop()
+        self.loop._keepaliveset.discard(self)
+        self._handle = None
+        self._watcher_set_data(self._watcher, self._FFI.NULL) # pylint:disable=no-member
+        self.callback = None
+        self.args = None
+
+    def _get_priority(self):
+        return None
+
+    @not_while_active
+    def _set_priority(self, priority):
+        pass
+
+    priority = property(_get_priority, _set_priority)
+
+
+    @property
+    def active(self):
+        if self._watcher is not None and self._watcher_is_active(self._watcher):
+            return True
+        return False
+
+    @property
+    def pending(self):
+        return False
+
+watcher = AbstractWatcherType('watcher', (object,), dict(watcher.__dict__))
+
+class IoMixin(object):
+
+    EVENT_MASK = 0
+
+    def __init__(self, loop, fd, events, ref=True, priority=None, _args=None):
+        # Win32 only works with sockets, and only when we use libuv, because
+        # we don't use _open_osfhandle. See libuv/watchers.py:io for a description.
+        if fd < 0:
+            raise ValueError('fd must be non-negative: %r' % fd)
+        if events & ~self.EVENT_MASK:
+            raise ValueError('illegal event mask: %r' % events)
+        self._fd = fd
+        super(IoMixin, self).__init__(loop, ref=ref, priority=priority,
+                                      args=_args or (fd, events))
+
+    def start(self, callback, *args, **kwargs):
+        args = args or _NOARGS
+        if kwargs.get('pass_events'):
+            args = (GEVENT_CORE_EVENTS, ) + args
+        super(IoMixin, self).start(callback, *args)
+
+    def _format(self):
+        return ' fd=%d' % self._fd
+
+class TimerMixin(object):
+    _watcher_type = 'timer'
+
+    def __init__(self, loop, after=0.0, repeat=0.0, ref=True, priority=None):
+        if repeat < 0.0:
+            raise ValueError("repeat must be positive or zero: %r" % repeat)
+        self._after = after
+        self._repeat = repeat
+        super(TimerMixin, self).__init__(loop, ref=ref, priority=priority, args=(after, repeat))
+
+    def start(self, callback, *args, **kw):
+        update = kw.get("update", self.loop.starting_timer_may_update_loop_time)
+        if update:
+            # Quoth the libev doc: "This is a costly operation and is
+            # usually done automatically within ev_run(). This
+            # function is rarely useful, but when some event callback
+            # runs for a very long time without entering the event
+            # loop, updating libev's idea of the current time is a
+            # good idea."
+
+            # 1.3 changed the default for this to False *unless* the loop is
+            # running a callback; see libuv for details. Note that
+            # starting Timeout objects still sets this to true.
+
+            self.loop.update_now()
+        super(TimerMixin, self).start(callback, *args)
+
+    def again(self, callback, *args, **kw):
+        raise NotImplementedError()
+
+
+class SignalMixin(object):
+    _watcher_type = 'signal'
+
+    def __init__(self, loop, signalnum, ref=True, priority=None):
+        if signalnum < 1 or signalnum >= signalmodule.NSIG:
+            raise ValueError('illegal signal number: %r' % signalnum)
+        # still possible to crash on one of libev's asserts:
+        # 1) "libev: ev_signal_start called with illegal signal number"
+        #    EV_NSIG might be different from signal.NSIG on some platforms
+        # 2) "libev: a signal must not be attached to two different loops"
+        #    we probably could check that in LIBEV_EMBED mode, but not in general
+        self._signalnum = signalnum
+        super(SignalMixin, self).__init__(loop, ref=ref, priority=priority, args=(signalnum, ))
+
+
+class IdleMixin(object):
+    _watcher_type = 'idle'
+
+
+class PrepareMixin(object):
+    _watcher_type = 'prepare'
+
+
+class CheckMixin(object):
+    _watcher_type = 'check'
+
+
+class ForkMixin(object):
+    _watcher_type = 'fork'
+
+
+class AsyncMixin(object):
+    _watcher_type = 'async'
+
+    def send(self):
+        raise NotImplementedError()
+
+    @property
+    def pending(self):
+        raise NotImplementedError()
+
+
+class ChildMixin(object):
+
+    # hack for libuv which doesn't extend watcher
+    _CALL_SUPER_INIT = True
+
+    def __init__(self, loop, pid, trace=0, ref=True):
+        if not loop.default:
+            raise TypeError('child watchers are only available on the default loop')
+        loop.install_sigchld()
+        self._pid = pid
+        if self._CALL_SUPER_INIT:
+            super(ChildMixin, self).__init__(loop, ref=ref, args=(pid, trace))
+
+    def _format(self):
+        return ' pid=%r rstatus=%r' % (self.pid, self.rstatus)
+
+    @property
+    def pid(self):
+        return self._pid
+
+    @property
+    def rpid(self):
+        # The received pid, the result of the waitpid() call.
+        return self._rpid
+
+    _rpid = None
+    _rstatus = 0
+
+    @property
+    def rstatus(self):
+        return self._rstatus
+
+class StatMixin(object):
+
+    @staticmethod
+    def _encode_path(path):
+        return fsencode(path)
+
+    def __init__(self, _loop, path, interval=0.0, ref=True, priority=None):
+        # Store the encoded path in the same attribute that corecext does
+        self._paths = self._encode_path(path)
+
+        # Keep the original path to avoid re-encoding, especially on Python 3
+        self._path = path
+
+        # Although CFFI would automatically convert a bytes object into a char* when
+        # calling ev_stat_init(..., char*, ...), on PyPy the char* pointer is not
+        # guaranteed to live past the function call. On CPython, only with a constant/interned
+        # bytes object is the pointer guaranteed to last path the function call. (And since
+        # Python 3 is pretty much guaranteed to produce a newly-encoded bytes object above, thats
+        # rarely the case). Therefore, we must keep a reference to the produced cdata object
+        # so that the struct ev_stat_watcher's `path` pointer doesn't become invalid/deallocated
+        self._cpath = self._FFI.new('char[]', self._paths)
+
+        self._interval = interval
+        super(StatMixin, self).__init__(_loop, ref=ref, priority=priority,
+                                        args=(self._cpath,
+                                              interval))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def attr(self):
+        raise NotImplementedError
+
+    @property
+    def prev(self):
+        raise NotImplementedError
+
+    @property
+    def interval(self):
+        return self._interval

python/gevent/_fileobjectcommon.py

@@ -1,10 +1,20 @@
+from __future__ import absolute_import, print_function, division
 
 try:
     from errno import EBADF
 except ImportError:
     EBADF = 9
 
+import os
 from io import TextIOWrapper
+import functools
+import sys
+
+
+from gevent.hub import _get_hub_noargs as get_hub
+from gevent._compat import integer_types
+from gevent._compat import reraise
+from gevent.lock import Semaphore, DummySemaphore
 
 class cancel_wait_ex(IOError):
 
@@ -53,6 +63,9 @@ class FileObjectBase(object):
     # Whether we are translating universal newlines or not.
     _translate = False
 
+    _translate_encoding = None
+    _translate_errors = None
+
     def __init__(self, io, closefd):
         """
         :param io: An io.IOBase-like object.
@@ -63,8 +76,9 @@ class FileObjectBase(object):
         self._close = closefd
 
         if self._translate:
-            # This automatically handles delegation.
-            self.translate_newlines(None)
+            # This automatically handles delegation by assigning to
+            # self.io
+            self.translate_newlines(None, self._translate_encoding, self._translate_errors)
         else:
             self._do_delegate_methods()
 
@@ -126,3 +140,136 @@ class FileObjectBase(object):
 
     def _extra_repr(self):
         return ''
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+class FileObjectBlock(FileObjectBase):
+
+    def __init__(self, fobj, *args, **kwargs):
+        closefd = kwargs.pop('close', True)
+        if kwargs:
+            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectBlock does not support close=False on an fd.')
+            fobj = os.fdopen(fobj, *args)
+        super(FileObjectBlock, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        fobj.close()
+
+class FileObjectThread(FileObjectBase):
+    """
+    A file-like object wrapping another file-like object, performing all blocking
+    operations on that object in a background thread.
+
+    .. caution::
+        Attempting to change the threadpool or lock of an existing FileObjectThread
+        has undefined consequences.
+
+    .. versionchanged:: 1.1b1
+       The file object is closed using the threadpool. Note that whether or
+       not this action is synchronous or asynchronous is not documented.
+
+    """
+
+    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
+        """
+        :param fobj: The underlying file-like object to wrap, or an integer fileno
+           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
+        :keyword bool lock: If True (the default) then all operations will
+           be performed one-by-one. Note that this does not guarantee that, if using
+           this file object from multiple threads/greenlets, operations will be performed
+           in any particular order, only that no two operations will be attempted at the
+           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
+           file operations with an external resource.
+        :keyword bool close: If True (the default) then when this object is closed,
+           the underlying object is closed as well.
+        """
+        closefd = close
+        self.threadpool = threadpool or get_hub().threadpool
+        self.lock = lock
+        if self.lock is True:
+            self.lock = Semaphore()
+        elif not self.lock:
+            self.lock = DummySemaphore()
+        if not hasattr(self.lock, '__enter__'):
+            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectThread does not support close=False on an fd.')
+            if mode is None:
+                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
+                fobj = os.fdopen(fobj)
+            else:
+                fobj = os.fdopen(fobj, mode, bufsize)
+
+        self.__io_holder = [fobj] # signal for _wrap_method
+        super(FileObjectThread, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        self.__io_holder[0] = None # for _wrap_method
+        try:
+            with self.lock:
+                self.threadpool.apply(fobj.flush)
+        finally:
+            if closefd:
+                # Note that we're not taking the lock; older code
+                # did fobj.close() without going through the threadpool at all,
+                # so acquiring the lock could potentially introduce deadlocks
+                # that weren't present before. Avoiding the lock doesn't make
+                # the existing race condition any worse.
+                # We wrap the close in an exception handler and re-raise directly
+                # to avoid the (common, expected) IOError from being logged by the pool
+                def close():
+                    try:
+                        fobj.close()
+                    except: # pylint:disable=bare-except
+                        return sys.exc_info()
+                exc_info = self.threadpool.apply(close)
+                if exc_info:
+                    reraise(*exc_info)
+
+    def _do_delegate_methods(self):
+        super(FileObjectThread, self)._do_delegate_methods()
+        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
+            self.read1 = self.read
+        self.__io_holder[0] = self._io
+
+    def _extra_repr(self):
+        return ' threadpool=%r' % (self.threadpool,)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        line = self.readline()
+        if line:
+            return line
+        raise StopIteration
+    __next__ = next
+
+    def _wrap_method(self, method):
+        # NOTE: We are careful to avoid introducing a refcycle
+        # within self. Our wrapper cannot refer to self.
+        io_holder = self.__io_holder
+        lock = self.lock
+        threadpool = self.threadpool
+
+        @functools.wraps(method)
+        def thread_method(*args, **kwargs):
+            if io_holder[0] is None:
+                # This is different than FileObjectPosix, etc,
+                # because we want to save the expensive trip through
+                # the threadpool.
+                raise FileObjectClosed()
+            with lock:
+                return threadpool.apply(method, args, kwargs)
+
+        return thread_method

python/gevent/_fileobjectposix.py

@@ -26,31 +26,30 @@ class GreenFileDescriptorIO(RawIOBase):
 
     _read_event = None
     _write_event = None
+    _closed = False
+    _seekable = None
 
     def __init__(self, fileno, mode='r', closefd=True):
         RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called
-        self._closed = False
         self._closefd = closefd
         self._fileno = fileno
         make_nonblocking(fileno)
-        self._readable = 'r' in mode
-        self._writable = 'w' in mode
+        readable = 'r' in mode
+        writable = 'w' in mode
         self.hub = get_hub()
 
         io_watcher = self.hub.loop.io
-        if self._readable:
+        if readable:
             self._read_event = io_watcher(fileno, 1)
 
-        if self._writable:
+        if writable:
             self._write_event = io_watcher(fileno, 2)
 
-        self._seekable = None
-
     def readable(self):
-        return self._readable
+        return self._read_event is not None
 
     def writable(self):
-        return self._writable
+        return self._write_event is not None
 
     def seekable(self):
         if self._seekable is None:
@@ -73,11 +72,18 @@ class GreenFileDescriptorIO(RawIOBase):
         if self._closed:
             return
         self.flush()
+        # TODO: Can we use 'read_event is not None and write_event is
+        # not None' to mean _closed?
         self._closed = True
-        if self._readable:
-            self.hub.cancel_wait(self._read_event, cancel_wait_ex)
-        if self._writable:
-            self.hub.cancel_wait(self._write_event, cancel_wait_ex)
+        read_event = self._read_event
+        write_event = self._write_event
+        self._read_event = self._write_event = None
+
+        if read_event is not None:
+            self.hub.cancel_wait(read_event, cancel_wait_ex, True)
+        if write_event is not None:
+            self.hub.cancel_wait(write_event, cancel_wait_ex, True)
+
         fileno = self._fileno
         if self._closefd:
             self._fileno = None
@@ -88,10 +94,10 @@ class GreenFileDescriptorIO(RawIOBase):
     # want to take advantage of this to avoid single byte reads when
     # possible. This is highlighted by a bug in BufferedIOReader that
     # calls read() in a loop when its readall() method is invoked;
-    # this was fixed in Python 3.3. See
+    # this was fixed in Python 3.3, but we still need our workaround for 2.7. See
     # https://github.com/gevent/gevent/issues/675)
     def __read(self, n):
-        if not self._readable:
+        if self._read_event is None:
             raise UnsupportedOperation('read')
         while True:
             try:
@@ -123,7 +129,7 @@ class GreenFileDescriptorIO(RawIOBase):
         return n
 
     def write(self, b):
-        if not self._writable:
+        if self._write_event is None:
             raise UnsupportedOperation('write')
         while True:
             try:
@@ -206,12 +212,20 @@ class FileObjectPosix(FileObjectBase):
             put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
         :keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
             (where the "b" or "U" can be omitted).
-            If "U" is part of the mode, IO will be done on text, otherwise bytes.
+            If "U" is part of the mode, universal newlines will be used. On Python 2,
+            if 't' is not in the mode, this will result in returning byte (native) strings;
+            putting 't'  in the mode will return text strings. This may cause
+            :exc:`UnicodeDecodeError` to be raised.
         :keyword int bufsize: If given, the size of the buffer to use. The default
             value means to use a platform-specific default
             Other values are interpreted as for the :mod:`io` package.
             Buffering is ignored in text mode.
 
+        .. versionchanged:: 1.3a1
+
+           On Python 2, enabling universal newlines no longer forces unicode
+           IO.
+
         .. versionchanged:: 1.2a1
 
            A bufsize of 0 in write mode is no longer forced to be 1.
@@ -234,10 +248,31 @@ class FileObjectPosix(FileObjectBase):
         mode = (mode or 'rb').replace('b', '')
         if 'U' in mode:
             self._translate = True
+            if bytes is str and 't' not in mode:
+                # We're going to be producing unicode objects, but
+                # universal newlines doesn't do that in the stdlib,
+                # so fix that to return str objects. The fix is two parts:
+                # first, set an encoding on the stream that can round-trip
+                # all bytes, and second, decode all bytes once they've been read.
+                self._translate_encoding = 'latin-1'
+                import functools
+
+                def wrap_method(m):
+                    if m.__name__.startswith("read"):
+                        @functools.wraps(m)
+                        def wrapped(*args, **kwargs):
+                            result = m(*args, **kwargs)
+                            assert isinstance(result, unicode) # pylint:disable=undefined-variable
+                            return result.encode('latin-1')
+                        return wrapped
+                    return m
+                self._wrap_method = wrap_method
             mode = mode.replace('U', '')
         else:
             self._translate = False
 
+        mode = mode.replace('t', '')
+
         if len(mode) != 1 and mode not in 'rw': # pragma: no cover
             # Python 3 builtin `open` raises a ValueError for invalid modes;
             # Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was

python/gevent/_greenlet.pxd

@@ -0,0 +1,174 @@
+# cython: auto_pickle=False
+
+cimport cython
+from gevent.__ident cimport IdentRegistry
+from gevent.__hub_local cimport get_hub_noargs as get_hub
+from gevent.__waiter cimport Waiter
+
+cdef bint _PYPY
+cdef sys_getframe
+cdef sys_exc_info
+cdef Timeout
+cdef GreenletExit
+cdef InvalidSwitchError
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+@cython.final
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef bint _greenlet_imported
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+cdef extern from "Python.h":
+
+    ctypedef class types.CodeType [object PyCodeObject]:
+        pass
+
+cdef extern from "frameobject.h":
+
+    ctypedef class types.FrameType [object PyFrameObject]:
+        cdef CodeType f_code
+        cdef int f_lineno
+        # We can't declare this in the object, because it's
+        # allowed to be NULL, and Cython can't handle that.
+        # We have to go through the python machinery to get a
+        # proper None instead.
+        # cdef FrameType f_back
+
+cdef void _init()
+
+cdef class SpawnedLink:
+    cdef public object callback
+
+
+@cython.final
+cdef class SuccessSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+cdef class FailureSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+@cython.internal
+@cython.freelist(1000)
+cdef class _Frame:
+    cdef readonly CodeType f_code
+    cdef readonly int f_lineno
+    cdef readonly _Frame f_back
+
+
+@cython.final
+@cython.locals(frames=list,frame=FrameType)
+cdef inline list _extract_stack(int limit)
+
+@cython.final
+@cython.locals(previous=_Frame, frame=tuple, f=_Frame)
+cdef _Frame _Frame_from_list(list frames)
+
+
+cdef class Greenlet(greenlet):
+    cdef readonly object value
+    cdef readonly tuple args
+    cdef readonly dict kwargs
+    cdef readonly object spawning_greenlet
+    cdef public dict spawn_tree_locals
+
+    # This is accessed with getattr() dynamically so it
+    # must be visible to Python
+    cdef readonly list _spawning_stack_frames
+
+    cdef list _links
+    cdef tuple _exc_info
+    cdef object _notifier
+    cdef object _start_event
+    cdef str _formatted_info
+    cdef object _ident
+
+    cpdef bint has_links(self)
+    cpdef join(self, timeout=*)
+    cpdef bint ready(self)
+    cpdef bint successful(self)
+    cpdef rawlink(self, object callback)
+    cpdef str _formatinfo(self)
+
+    @cython.locals(reg=IdentRegistry)
+    cdef _get_minimal_ident(self)
+
+
+    cdef bint __started_but_aborted(self)
+    cdef bint __start_cancelled_by_kill(self)
+    cdef bint __start_pending(self)
+    cdef bint __never_started_or_killed(self)
+    cdef bint __start_completed(self)
+    cdef __handle_death_before_start(self, tuple args)
+
+    cdef __cancel_start(self)
+
+    cdef _report_result(self, object result)
+    cdef _report_error(self, tuple exc_info)
+    # This is used as the target of a callback
+    # from the loop, and so needs to be a cpdef
+    cpdef _notify_links(self)
+
+    # Hmm, declaring _raise_exception causes issues when _imap
+    # is also compiled.
+    # TypeError: wrap() takes exactly one argument (0 given)
+    # cpdef _raise_exception(self)
+
+
+
+# Declare a bunch of imports as cdefs so they can
+# be accessed directly as static vars without
+# doing a module global lookup. This is especially important
+# for spawning greenlets.
+cdef _greenlet__init__
+cdef _threadlocal
+cdef get_hub_class
+cdef wref
+
+cdef dump_traceback
+cdef load_traceback
+cdef Waiter
+cdef wait
+cdef iwait
+cdef reraise
+cpdef GEVENT_CONFIG
+
+
+@cython.final
+@cython.internal
+cdef class _dummy_event:
+    cdef readonly bint pending
+    cdef readonly bint active
+
+    cpdef stop(self)
+    cpdef start(self, cb)
+    cpdef close(self)
+
+cdef _dummy_event _cancelled_start_event
+cdef _dummy_event _start_completed_event
+
+
+@cython.locals(diehards=list)
+cdef _killall3(list greenlets, object exception, object waiter)
+cdef _killall(list greenlets, object exception)
+
+@cython.locals(done=list)
+cpdef joinall(greenlets, timeout=*, raise_error=*, count=*)

python/gevent/_greenlet_primitives.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# copyright (c) 2018 gevent. See  LICENSE.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+"""
+A collection of primitives used by the hub, and suitable for
+compilation with Cython because of their frequency of use.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from weakref import ref as wref
+
+from greenlet import greenlet
+
+from gevent.exceptions import BlockingSwitchOutError
+
+
+# In Cython, we define these as 'cdef inline' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+locals()['_greenlet_switch'] = greenlet.switch
+
+__all__ = [
+    'TrackedRawGreenlet',
+    'SwitchOutGreenletWithLoop',
+]
+
+class TrackedRawGreenlet(greenlet):
+
+    def __init__(self, function, parent):
+        greenlet.__init__(self, function, parent)
+        # See greenlet.py's Greenlet class. We capture the cheap
+        # parts to maintain the tree structure, but we do not capture
+        # the stack because that's too expensive for 'spawn_raw'.
+
+        current = getcurrent() # pylint:disable=undefined-variable
+        self.spawning_greenlet = wref(current)
+        # See Greenlet for how trees are maintained.
+        try:
+            self.spawn_tree_locals = current.spawn_tree_locals
+        except AttributeError:
+            self.spawn_tree_locals = {}
+            if current.parent:
+                current.spawn_tree_locals = self.spawn_tree_locals
+
+
+class SwitchOutGreenletWithLoop(TrackedRawGreenlet):
+    # Subclasses must define:
+    # - self.loop
+
+    # This class defines loop in its .pxd for Cython. This lets us avoid
+    # circular dependencies with the hub.
+
+    def switch(self):
+        switch_out = getattr(getcurrent(), 'switch_out', None) # pylint:disable=undefined-variable
+        if switch_out is not None:
+            switch_out()
+        return _greenlet_switch(self) # pylint:disable=undefined-variable
+
+    def switch_out(self):
+        raise BlockingSwitchOutError('Impossible to call blocking function in the event loop callback')
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__greenlet_primitives')

python/gevent/_hub_local.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# copyright 2018 gevent. See LICENSE
+"""
+Maintains the thread local hub.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from gevent._compat import thread_mod_name
+
+__all__ = [
+    'get_hub',
+    'get_hub_noargs',
+    'get_hub_if_exists',
+]
+
+# These must be the "real" native thread versions,
+# not monkey-patched.
+# We are imported early enough (by gevent/__init__) that
+# we can rely on not being monkey-patched in any way yet.
+class _Threadlocal(__import__(thread_mod_name)._local):
+
+    def __init__(self):
+        # Use a class with an initializer so that we can test
+        # for 'is None' instead of catching AttributeError, making
+        # the code cleaner and possibly solving some corner cases
+        # (like #687)
+        super(_Threadlocal, self).__init__()
+        self.Hub = None
+        self.loop = None
+        self.hub = None
+
+_threadlocal = _Threadlocal()
+
+Hub = None # Set when gevent.hub is imported
+
+def get_hub_class():
+    """Return the type of hub to use for the current thread.
+
+    If there's no type of hub for the current thread yet, 'gevent.hub.Hub' is used.
+    """
+    hubtype = _threadlocal.Hub
+    if hubtype is None:
+        hubtype = _threadlocal.Hub = Hub
+    return hubtype
+
+def set_default_hub_class(hubtype):
+    global Hub
+    Hub = hubtype
+
+def get_hub(*args, **kwargs):
+    """
+    Return the hub for the current thread.
+
+    If a hub does not exist in the current thread, a new one is
+    created of the type returned by :func:`get_hub_class`.
+
+    .. deprecated:: 1.3b1
+       The ``*args`` and ``**kwargs`` arguments are deprecated. They were
+       only used when the hub was created, and so were non-deterministic---to be
+       sure they were used, *all* callers had to pass them, or they were order-dependent.
+       Use ``set_hub`` instead.
+    """
+    hub = _threadlocal.hub
+    if hub is None:
+        hubtype = get_hub_class()
+        hub = _threadlocal.hub = hubtype(*args, **kwargs)
+    return hub
+
+def get_hub_noargs():
+    # Just like get_hub, but cheaper to call because it
+    # takes no arguments or kwargs. See also a copy in
+    # gevent/greenlet.py
+    hub = _threadlocal.hub
+    if hub is None:
+        hubtype = get_hub_class()
+        hub = _threadlocal.hub = hubtype()
+    return hub
+
+def get_hub_if_exists():
+    """Return the hub for the current thread.
+
+    Return ``None`` if no hub has been created yet.
+    """
+    return _threadlocal.hub
+
+
+def set_hub(hub):
+    _threadlocal.hub = hub
+
+def get_loop():
+    return _threadlocal.loop
+
+def set_loop(loop):
+    _threadlocal.loop = loop
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__hub_local')

python/gevent/_hub_primitives.py

@@ -0,0 +1,371 @@
+# -*- coding: utf-8 -*-
+# copyright (c) 2018 gevent. See  LICENSE.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,binding=True
+"""
+A collection of primitives used by the hub, and suitable for
+compilation with Cython because of their frequency of use.
+
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import traceback
+
+from gevent.exceptions import InvalidSwitchError
+from gevent.exceptions import ConcurrentObjectUseError
+
+from gevent import _greenlet_primitives
+from gevent import _waiter
+from gevent._util import _NONE
+from gevent._hub_local import get_hub_noargs as get_hub
+from gevent.timeout import Timeout
+
+# In Cython, we define these as 'cdef inline' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+locals()['Waiter'] = _waiter.Waiter
+locals()['MultipleWaiter'] = _waiter.MultipleWaiter
+locals()['SwitchOutGreenletWithLoop'] = _greenlet_primitives.SwitchOutGreenletWithLoop
+
+__all__ = [
+    'WaitOperationsGreenlet',
+    'iwait_on_objects',
+    'wait_on_objects',
+    'wait_read',
+    'wait_write',
+    'wait_readwrite',
+]
+
+class WaitOperationsGreenlet(SwitchOutGreenletWithLoop): # pylint:disable=undefined-variable
+
+    def wait(self, watcher):
+        """
+        Wait until the *watcher* (which must not be started) is ready.
+
+        The current greenlet will be unscheduled during this time.
+        """
+        waiter = Waiter(self) # pylint:disable=undefined-variable
+        watcher.start(waiter.switch, waiter)
+        try:
+            result = waiter.get()
+            if result is not waiter:
+                raise InvalidSwitchError('Invalid switch into %s: %r (expected %r)' % (
+                    getcurrent(), # pylint:disable=undefined-variable
+                    result, waiter))
+        finally:
+            watcher.stop()
+
+    def cancel_wait(self, watcher, error, close_watcher=False):
+        """
+        Cancel an in-progress call to :meth:`wait` by throwing the given *error*
+        in the waiting greenlet.
+
+        .. versionchanged:: 1.3a1
+           Added the *close_watcher* parameter. If true, the watcher
+           will be closed after the exception is thrown. The watcher should then
+           be discarded. Closing the watcher is important to release native resources.
+        .. versionchanged:: 1.3a2
+           Allow the *watcher* to be ``None``. No action is taken in that case.
+        """
+        if watcher is None:
+            # Presumably already closed.
+            # See https://github.com/gevent/gevent/issues/1089
+            return
+        if watcher.callback is not None:
+            self.loop.run_callback(self._cancel_wait, watcher, error, close_watcher)
+        elif close_watcher:
+            watcher.close()
+
+    def _cancel_wait(self, watcher, error, close_watcher):
+        # We have to check again to see if it was still active by the time
+        # our callback actually runs.
+        active = watcher.active
+        cb = watcher.callback
+        if close_watcher:
+            watcher.close()
+        if active:
+            # The callback should be greenlet.switch(). It may or may not be None.
+            glet = getattr(cb, '__self__', None)
+            if glet is not None:
+                glet.throw(error)
+
+
+class _WaitIterator(object):
+
+    def __init__(self, objects, hub, timeout, count):
+        self._hub = hub
+        self._waiter = MultipleWaiter(hub) # pylint:disable=undefined-variable
+        self._switch = self._waiter.switch
+        self._timeout = timeout
+        self._objects = objects
+
+        self._timer = None
+        self._begun = False
+
+
+        # Even if we're only going to return 1 object,
+        # we must still rawlink() *all* of them, so that no
+        # matter which one finishes first we find it.
+        self._count = len(objects) if count is None else min(count, len(objects))
+
+
+    def __iter__(self):
+        # When we begin iterating, we begin the timer.
+        # XXX: If iteration doesn't actually happen, we
+        # could leave these links around!
+        if not self._begun:
+            self._begun = True
+
+            for obj in self._objects:
+                obj.rawlink(self._switch)
+
+            if self._timeout is not None:
+                self._timer = self._hub.loop.timer(self._timeout, priority=-1)
+                self._timer.start(self._switch, self)
+        return self
+
+    def __next__(self):
+        if self._count == 0:
+            # Exhausted
+            self._cleanup()
+            raise StopIteration()
+
+        self._count -= 1
+        try:
+            item = self._waiter.get()
+            self._waiter.clear()
+            if item is self:
+                # Timer expired, no more
+                self._cleanup()
+                raise StopIteration()
+            return item
+        except:
+            self._cleanup()
+            raise
+
+    next = __next__
+
+    def _cleanup(self):
+        if self._timer is not None:
+            self._timer.close()
+            self._timer = None
+
+        objs = self._objects
+        self._objects = ()
+        for aobj in objs:
+            unlink = getattr(aobj, 'unlink', None)
+            if unlink is not None:
+                try:
+                    unlink(self._switch)
+                except: # pylint:disable=bare-except
+                    traceback.print_exc()
+
+
+def iwait_on_objects(objects, timeout=None, count=None):
+    """
+    Iteratively yield *objects* as they are ready, until all (or *count*) are ready
+    or *timeout* expired.
+
+    :param objects: A sequence (supporting :func:`len`) containing objects
+        implementing the wait protocol (rawlink() and unlink()).
+    :keyword int count: If not `None`, then a number specifying the maximum number
+        of objects to wait for. If ``None`` (the default), all objects
+        are waited for.
+    :keyword float timeout: If given, specifies a maximum number of seconds
+        to wait. If the timeout expires before the desired waited-for objects
+        are available, then this method returns immediately.
+
+    .. seealso:: :func:`wait`
+
+    .. versionchanged:: 1.1a1
+       Add the *count* parameter.
+    .. versionchanged:: 1.1a2
+       No longer raise :exc:`LoopExit` if our caller switches greenlets
+       in between items yielded by this function.
+    """
+    # QQQ would be nice to support iterable here that can be generated slowly (why?)
+    hub = get_hub()
+    if objects is None:
+        return [hub.join(timeout=timeout)]
+    return _WaitIterator(objects, hub, timeout, count)
+
+
+def wait_on_objects(objects=None, timeout=None, count=None):
+    """
+    Wait for ``objects`` to become ready or for event loop to finish.
+
+    If ``objects`` is provided, it must be a list containing objects
+    implementing the wait protocol (rawlink() and unlink() methods):
+
+    - :class:`gevent.Greenlet` instance
+    - :class:`gevent.event.Event` instance
+    - :class:`gevent.lock.Semaphore` instance
+    - :class:`gevent.subprocess.Popen` instance
+
+    If ``objects`` is ``None`` (the default), ``wait()`` blocks until
+    the current event loop has nothing to do (or until ``timeout`` passes):
+
+    - all greenlets have finished
+    - all servers were stopped
+    - all event loop watchers were stopped.
+
+    If ``count`` is ``None`` (the default), wait for all ``objects``
+    to become ready.
+
+    If ``count`` is a number, wait for (up to) ``count`` objects to become
+    ready. (For example, if count is ``1`` then the function exits
+    when any object in the list is ready).
+
+    If ``timeout`` is provided, it specifies the maximum number of
+    seconds ``wait()`` will block.
+
+    Returns the list of ready objects, in the order in which they were
+    ready.
+
+    .. seealso:: :func:`iwait`
+    """
+    if objects is None:
+        hub = get_hub()
+        return hub.join(timeout=timeout) # pylint:disable=
+    return list(iwait_on_objects(objects, timeout, count))
+
+_timeout_error = Exception
+
+def set_default_timeout_error(e):
+    global _timeout_error
+    _timeout_error = e
+
+def _primitive_wait(watcher, timeout, timeout_exc, hub):
+    if watcher.callback is not None:
+        raise ConcurrentObjectUseError('This socket is already used by another greenlet: %r'
+                                       % (watcher.callback, ))
+
+    if hub is None:
+        hub = get_hub()
+
+    if timeout is None:
+        hub.wait(watcher)
+        return
+
+    timeout = Timeout._start_new_or_dummy(
+        timeout,
+        (timeout_exc
+         if timeout_exc is not _NONE or timeout is None
+         else _timeout_error('timed out')))
+
+    with timeout:
+        hub.wait(watcher)
+
+# Suitable to be bound as an instance method
+def wait_on_socket(socket, watcher, timeout_exc=None):
+    _primitive_wait(watcher, socket.timeout,
+                    timeout_exc if timeout_exc is not None else _NONE,
+                    socket.hub)
+
+def wait_on_watcher(watcher, timeout=None, timeout_exc=_NONE, hub=None):
+    """
+    wait(watcher, timeout=None, [timeout_exc=None]) -> None
+
+    Block the current greenlet until *watcher* is ready.
+
+    If *timeout* is non-negative, then *timeout_exc* is raised after
+    *timeout* second has passed.
+
+    If :func:`cancel_wait` is called on *io* by another greenlet,
+    raise an exception in this blocking greenlet
+    (``socket.error(EBADF, 'File descriptor was closed in another
+    greenlet')`` by default).
+
+    :param io: An event loop watcher, most commonly an IO watcher obtained from
+        :meth:`gevent.core.loop.io`
+    :keyword timeout_exc: The exception to raise if the timeout expires.
+        By default, a :class:`socket.timeout` exception is raised.
+        If you pass a value for this keyword, it is interpreted as for
+        :class:`gevent.timeout.Timeout`.
+
+    :raises ~gevent.hub.ConcurrentObjectUseError: If the *watcher* is
+        already started.
+    """
+    _primitive_wait(watcher, timeout, timeout_exc, hub)
+
+
+def wait_read(fileno, timeout=None, timeout_exc=_NONE):
+    """
+    wait_read(fileno, timeout=None, [timeout_exc=None]) -> None
+
+    Block the current greenlet until *fileno* is ready to read.
+
+    For the meaning of the other parameters and possible exceptions,
+    see :func:`wait`.
+
+    .. seealso:: :func:`cancel_wait`
+    """
+    hub = get_hub()
+    io = hub.loop.io(fileno, 1)
+    try:
+        return wait_on_watcher(io, timeout, timeout_exc, hub)
+    finally:
+        io.close()
+
+
+def wait_write(fileno, timeout=None, timeout_exc=_NONE, event=_NONE):
+    """
+    wait_write(fileno, timeout=None, [timeout_exc=None]) -> None
+
+    Block the current greenlet until *fileno* is ready to write.
+
+    For the meaning of the other parameters and possible exceptions,
+    see :func:`wait`.
+
+    .. deprecated:: 1.1
+       The keyword argument *event* is ignored. Applications should not pass this parameter.
+       In the future, doing so will become an error.
+
+    .. seealso:: :func:`cancel_wait`
+    """
+    # pylint:disable=unused-argument
+    hub = get_hub()
+    io = hub.loop.io(fileno, 2)
+    try:
+        return wait_on_watcher(io, timeout, timeout_exc, hub)
+    finally:
+        io.close()
+
+
+def wait_readwrite(fileno, timeout=None, timeout_exc=_NONE, event=_NONE):
+    """
+    wait_readwrite(fileno, timeout=None, [timeout_exc=None]) -> None
+
+    Block the current greenlet until *fileno* is ready to read or
+    write.
+
+    For the meaning of the other parameters and possible exceptions,
+    see :func:`wait`.
+
+    .. deprecated:: 1.1
+       The keyword argument *event* is ignored. Applications should not pass this parameter.
+       In the future, doing so will become an error.
+
+    .. seealso:: :func:`cancel_wait`
+    """
+    # pylint:disable=unused-argument
+    hub = get_hub()
+    io = hub.loop.io(fileno, 3)
+    try:
+        return wait_on_watcher(io, timeout, timeout_exc, hub)
+    finally:
+        io.close()
+
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__hub_primitives')

python/gevent/_ident.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 gevent contributors. See LICENSE for details.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from weakref import WeakKeyDictionary
+from weakref import ref
+
+from heapq import heappop
+from heapq import heappush
+
+__all__ = [
+    'IdentRegistry',
+]
+
+class ValuedWeakRef(ref):
+    """
+    A weak ref with an associated value.
+    """
+    # This seems entirely spurious; even on Python 2.7
+    # weakref.ref descends from object
+    # pylint: disable=slots-on-old-class
+    __slots__ = ('value',)
+
+
+class IdentRegistry(object):
+    """
+    Maintains a unique mapping of (small) positive integer identifiers
+    to objects that can be weakly referenced.
+
+    It is guaranteed that no two objects will have the the same
+    identifier at the same time, as long as those objects are
+    also uniquely hashable.
+    """
+
+    def __init__(self):
+        # {obj -> (ident, wref(obj))}
+        self._registry = WeakKeyDictionary()
+
+        # A heap of numbers that have been used and returned
+        self._available_idents = []
+
+    def get_ident(self, obj):
+        """
+        Retrieve the identifier for *obj*, creating one
+        if necessary.
+        """
+
+        try:
+            return self._registry[obj][0]
+        except KeyError:
+            pass
+
+        if self._available_idents:
+            # Take the smallest free number
+            ident = heappop(self._available_idents)
+        else:
+            # Allocate a bigger one
+            ident = len(self._registry)
+
+        vref = ValuedWeakRef(obj, self._return_ident)
+        vref.value = ident # pylint:disable=assigning-non-slot,attribute-defined-outside-init
+        self._registry[obj] = (ident, vref)
+        return ident
+
+    def _return_ident(self, vref):
+        # By the time this is called, self._registry has been
+        # updated
+        if heappush is not None:
+            # Under some circumstances we can get called
+            # when the interpreter is shutting down, and globals
+            # aren't available any more.
+            heappush(self._available_idents, vref.value)
+
+    def __len__(self):
+        return len(self._registry)
+
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__ident')

python/gevent/_imap.py

@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2018 gevent
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,infer_types=True
+
+"""
+Iterators across greenlets or AsyncResult objects.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from gevent import _semaphore
+from gevent import queue
+
+
+__all__ = [
+    'IMapUnordered',
+    'IMap',
+]
+
+locals()['Greenlet'] = __import__('gevent').Greenlet
+locals()['Semaphore'] = _semaphore.Semaphore
+locals()['UnboundQueue'] = queue.UnboundQueue
+
+
+class Failure(object):
+    __slots__ = ('exc', 'raise_exception')
+
+    def __init__(self, exc, raise_exception=None):
+        self.exc = exc
+        self.raise_exception = raise_exception
+
+
+def _raise_exc(failure):
+    # For cython.
+    if failure.raise_exception:
+        failure.raise_exception()
+    else:
+        raise failure.exc
+
+class IMapUnordered(Greenlet): # pylint:disable=undefined-variable
+    """
+    At iterator of map results.
+    """
+
+    def __init__(self, func, iterable, spawn, maxsize=None, _zipped=False):
+        """
+        An iterator that.
+
+        :param callable spawn: The function we use to create new greenlets.
+        :keyword int maxsize: If given and not-None, specifies the maximum number of
+            finished results that will be allowed to accumulated awaiting the reader;
+            more than that number of results will cause map function greenlets to begin
+            to block. This is most useful is there is a great disparity in the speed of
+            the mapping code and the consumer and the results consume a great deal of resources.
+            Using a bound is more computationally expensive than not using a bound.
+
+        .. versionchanged:: 1.1b3
+            Added the *maxsize* parameter.
+        """
+        Greenlet.__init__(self) # pylint:disable=undefined-variable
+        self.spawn = spawn
+        self._zipped = _zipped
+        self.func = func
+        self.iterable = iterable
+        self.queue = UnboundQueue() # pylint:disable=undefined-variable
+
+
+        if maxsize:
+            # Bounding the queue is not enough if we want to keep from
+            # accumulating objects; the result value will be around as
+            # the greenlet's result, blocked on self.queue.put(), and
+            # we'll go on to spawn another greenlet, which in turn can
+            # create the result. So we need a semaphore to prevent a
+            # greenlet from exiting while the queue is full so that we
+            # don't spawn the next greenlet (assuming that self.spawn
+            # is of course bounded). (Alternatively we could have the
+            # greenlet itself do the insert into the pool, but that
+            # takes some rework).
+            #
+            # Given the use of a semaphore at this level, sizing the queue becomes
+            # redundant, and that lets us avoid having to use self.link() instead
+            # of self.rawlink() to avoid having blocking methods called in the
+            # hub greenlet.
+            self._result_semaphore = Semaphore(maxsize) # pylint:disable=undefined-variable
+        else:
+            self._result_semaphore = None
+
+        self._outstanding_tasks = 0
+        # The index (zero based) of the maximum number of
+        # results we will have.
+        self._max_index = -1
+        self.finished = False
+
+
+    # We're iterating in a different greenlet than we're running.
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self._result_semaphore is not None:
+            self._result_semaphore.release()
+        value = self._inext()
+        if isinstance(value, Failure):
+            _raise_exc(value)
+        return value
+
+    next = __next__ # Py2
+
+    def _inext(self):
+        return self.queue.get()
+
+    def _ispawn(self, func, item, item_index):
+        if self._result_semaphore is not None:
+            self._result_semaphore.acquire()
+        self._outstanding_tasks += 1
+        g = self.spawn(func, item) if not self._zipped else self.spawn(func, *item)
+        g._imap_task_index = item_index
+        g.rawlink(self._on_result)
+        return g
+
+    def _run(self): # pylint:disable=method-hidden
+        try:
+            func = self.func
+            for item in self.iterable:
+                self._max_index += 1
+                self._ispawn(func, item, self._max_index)
+            self._on_finish(None)
+        except BaseException as e:
+            self._on_finish(e)
+            raise
+        finally:
+            self.spawn = None
+            self.func = None
+            self.iterable = None
+            self._result_semaphore = None
+
+    def _on_result(self, greenlet):
+        # This method will be called in the hub greenlet (we rawlink)
+        self._outstanding_tasks -= 1
+        count = self._outstanding_tasks
+        finished = self.finished
+        ready = self.ready()
+        put_finished = False
+
+        if ready and count <= 0 and not finished:
+            finished = self.finished = True
+            put_finished = True
+
+        if greenlet.successful():
+            self.queue.put(self._iqueue_value_for_success(greenlet))
+        else:
+            self.queue.put(self._iqueue_value_for_failure(greenlet))
+
+        if put_finished:
+            self.queue.put(self._iqueue_value_for_self_finished())
+
+    def _on_finish(self, exception):
+        # Called in this greenlet.
+        if self.finished:
+            return
+
+        if exception is not None:
+            self.finished = True
+            self.queue.put(self._iqueue_value_for_self_failure(exception))
+            return
+
+        if self._outstanding_tasks <= 0:
+            self.finished = True
+            self.queue.put(self._iqueue_value_for_self_finished())
+
+    def _iqueue_value_for_success(self, greenlet):
+        return greenlet.value
+
+    def _iqueue_value_for_failure(self, greenlet):
+        return Failure(greenlet.exception, getattr(greenlet, '_raise_exception'))
+
+    def _iqueue_value_for_self_finished(self):
+        return Failure(StopIteration())
+
+    def _iqueue_value_for_self_failure(self, exception):
+        return Failure(exception, self._raise_exception)
+
+
+class IMap(IMapUnordered):
+    # A specialization of IMapUnordered that returns items
+    # in the order in which they were generated, not
+    # the order in which they finish.
+
+    def __init__(self, *args, **kwargs):
+        # The result dictionary: {index: value}
+        self._results = {}
+
+        # The index of the result to return next.
+        self.index = 0
+        IMapUnordered.__init__(self, *args, **kwargs)
+
+    def _inext(self):
+        try:
+            value = self._results.pop(self.index)
+        except KeyError:
+            # Wait for our index to finish.
+            while 1:
+                index, value = self.queue.get()
+                if index == self.index:
+                    break
+                else:
+                    self._results[index] = value
+        self.index += 1
+        return value
+
+    def _iqueue_value_for_success(self, greenlet):
+        return (greenlet._imap_task_index, IMapUnordered._iqueue_value_for_success(self, greenlet))
+
+    def _iqueue_value_for_failure(self, greenlet):
+        return (greenlet._imap_task_index, IMapUnordered._iqueue_value_for_failure(self, greenlet))
+
+    def _iqueue_value_for_self_finished(self):
+        return (self._max_index + 1, IMapUnordered._iqueue_value_for_self_finished(self))
+
+    def _iqueue_value_for_self_failure(self, exception):
+        return (self._max_index + 1, IMapUnordered._iqueue_value_for_self_failure(self, exception))
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__imap')

python/gevent/_interfaces.py

@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2018 gevent contributors. See LICENSE for details.
+"""
+Interfaces gevent uses that don't belong any one place.
+
+This is not a public module, these interfaces are not
+currently exposed to the public, they mostly exist for
+documentation and testing purposes.
+
+.. versionadded:: 1.3b2
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from gevent._util import Interface
+from gevent._util import Attribute
+
+# pylint:disable=no-method-argument, unused-argument, no-self-argument
+
+__all__ = [
+    'ILoop',
+    'IWatcher',
+]
+
+class ILoop(Interface):
+    """
+    The common interface expected for all event loops.
+
+    .. caution::
+       This is an internal, low-level interface. It may change
+       between minor versions of gevent.
+
+    .. rubric:: Watchers
+
+    The methods that create event loop watchers are `io`, `timer`,
+    `signal`, `idle`, `prepare`, `check`, `fork`, `async_`, `child`,
+    `stat`. These all return various types of :class:`IWatcher`.
+
+    All of those methods have one or two common arguments. *ref* is a
+    boolean saying whether the event loop is allowed to exit even if
+    this watcher is still started. *priority* is event loop specific.
+    """
+
+    default = Attribute("Boolean indicating whether this is the default loop")
+
+    def run(nowait=False, once=False):
+        """
+        Run the event loop.
+
+        This is usually called automatically by the hub greenlet, but
+        in special cases (when the hub is *not* running) you can use
+        this to control how the event loop runs (for example, to integrate
+        it with another event loop).
+        """
+
+    def now():
+        """
+        now() -> float
+
+        Return the loop's notion of the current time.
+
+        This may not necessarily be related to :func:`time.time` (it
+        may have a different starting point), but it must be expressed
+        in fractional seconds (the same *units* used by :func:`time.time`).
+        """
+
+    def update_now():
+        """
+        Update the loop's notion of the current time.
+
+        .. versionadded:: 1.3
+           In the past, this available as ``update``. This is still available as
+           an alias but will be removed in the future.
+        """
+
+    def destroy():
+        """
+        Clean up resources used by this loop.
+
+        If you create loops
+        (especially loops that are not the default) you *should* call
+        this method when you are done with the loop.
+
+        .. caution::
+
+            As an implementation note, the libev C loop implementation has a
+            finalizer (``__del__``) that destroys the object, but the libuv
+            and libev CFFI implementations do not. The C implementation may change.
+
+        """
+
+    def io(fd, events, ref=True, priority=None):
+        """
+        Create and return a new IO watcher for the given *fd*.
+
+        *events* is a bitmask specifying which events to watch
+        for. 1 means read, and 2 means write.
+        """
+
+    def timer(after, repeat=0.0, ref=True, priority=None):
+        """
+        Create and return a timer watcher that will fire after *after* seconds.
+
+        If *repeat* is given, the timer will continue to fire every *repeat* seconds.
+        """
+
+    def signal(signum, ref=True, priority=None):
+        """
+        Create and return a signal watcher for the signal *signum*,
+        one of the constants defined in :mod:`signal`.
+
+        This is platform and event loop specific.
+        """
+
+    def idle(ref=True, priority=None):
+        """
+        Create and return a watcher that fires when the event loop is idle.
+        """
+
+    def prepare(ref=True, priority=None):
+        """
+        Create and return a watcher that fires before the event loop
+        polls for IO.
+
+        .. caution:: This method is not supported by libuv.
+        """
+
+    def check(ref=True, priority=None):
+        """
+        Create and return a watcher that fires after the event loop
+        polls for IO.
+        """
+
+    def fork(ref=True, priority=None):
+        """
+        Create a watcher that fires when the process forks.
+
+        Availability: POSIX
+        """
+
+    def async_(ref=True, priority=None):
+        """
+        Create a watcher that fires when triggered, possibly
+        from another thread.
+
+        .. versionchanged:: 1.3
+           This was previously just named ``async``; for compatibility
+           with Python 3.7 where ``async`` is a keyword it was renamed.
+           On older versions of Python the old name is still around, but
+           it will be removed in the future.
+        """
+
+    def child(pid, trace=0, ref=True):
+        """
+        Create a watcher that fires for events on the child with process ID *pid*.
+
+        This is platform specific.
+        """
+
+    def stat(path, interval=0.0, ref=True, priority=None):
+        """
+        Create a watcher that monitors the filesystem item at *path*.
+
+        If the operating system doesn't support event notifications
+        from the filesystem, poll for changes every *interval* seconds.
+        """
+
+    def run_callback(func, *args):
+        """
+        Run the *func* passing it *args* at the next opportune moment.
+
+        This is a way of handing control to the event loop and deferring
+        an action.
+        """
+
+class IWatcher(Interface):
+    """
+    An event loop watcher.
+
+    These objects call their *callback* function when the event
+    loop detects the event has happened.
+
+    .. important:: You *must* call :meth:`close` when you are
+       done with this object to avoid leaking native resources.
+    """
+
+    def start(callback, *args, **kwargs):
+        """
+        Have the event loop begin watching for this event.
+
+        When the event is detected, *callback* will be called with
+        *args*.
+
+        .. caution::
+
+            Not all watchers accept ``**kwargs``,
+            and some watchers define special meanings for certain keyword args.
+        """
+
+    def stop():
+        """
+        Have the event loop stop watching this event.
+
+        In the future you may call :meth:`start` to begin watching
+        again.
+        """
+
+    def close():
+        """
+        Dispose of any native resources associated with the watcher.
+
+        If we were active, stop.
+
+        Attempting to operate on this object after calling close is
+        undefined. You should dispose of any references you have to it
+        after calling this method.
+        """

python/gevent/_local.pxd

@@ -0,0 +1,113 @@
+# cython: auto_pickle=False
+
+cimport cython
+from gevent._greenlet cimport Greenlet
+
+cdef bint _PYPY
+cdef ref
+cdef copy
+
+cdef object _marker
+cdef str key_prefix
+cdef bint _greenlet_imported
+
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+
+cdef void _init()
+
+@cython.final
+@cython.internal
+cdef class _wrefdict(dict):
+   cdef object __weakref__
+
+@cython.final
+@cython.internal
+cdef class _greenlet_deleted:
+    cdef object idt
+    cdef object wrdicts
+
+
+@cython.final
+@cython.internal
+cdef class _local_deleted:
+    cdef str key
+    cdef object wrthread
+    cdef _greenlet_deleted greenlet_deleted
+
+@cython.final
+@cython.internal
+cdef class _localimpl:
+    cdef str key
+    cdef dict dicts
+    cdef tuple localargs
+    cdef dict localkwargs
+    cdef tuple localtypeid
+    cdef object __weakref__
+
+
+@cython.final
+@cython.internal
+cdef class _localimpl_dict_entry:
+    cdef object wrgreenlet
+    cdef dict localdict
+
+@cython.locals(localdict=dict, key=str,
+               greenlet_deleted=_greenlet_deleted,
+               local_deleted=_local_deleted)
+cdef dict _localimpl_create_dict(_localimpl self,
+                                 greenlet greenlet,
+                                 object idt)
+
+cdef set _local_attrs
+
+cdef class local:
+    cdef _localimpl _local__impl
+    cdef set _local_type_get_descriptors
+    cdef set _local_type_set_or_del_descriptors
+    cdef set _local_type_del_descriptors
+    cdef set _local_type_set_descriptors
+    cdef set _local_type_vars
+    cdef type _local_type
+
+    @cython.locals(entry=_localimpl_dict_entry,
+                   dct=dict, duplicate=dict,
+                   instance=local)
+    cpdef local __copy__(local self)
+
+
+@cython.locals(impl=_localimpl,dct=dict,
+               dct=dict, entry=_localimpl_dict_entry)
+cdef inline dict _local_get_dict(local self)
+
+@cython.locals(entry=_localimpl_dict_entry)
+cdef _local__copy_dict_from(local self, _localimpl impl, dict duplicate)
+
+@cython.locals(mro=list, gets=set, dels=set, set_or_del=set,
+               type_self=type, type_attr=type,
+               sets=set)
+cdef tuple _local_find_descriptors(local self)
+
+@cython.locals(result=list, local_impl=_localimpl,
+               entry=_localimpl_dict_entry, k=str,
+               greenlet_dict=dict)
+cpdef all_local_dicts_for_greenlet(greenlet greenlet)

python/gevent/_monitor.py

@@ -0,0 +1,325 @@
+# Copyright (c) 2018 gevent. See LICENSE for details.
+from __future__ import print_function, absolute_import, division
+
+import os
+import sys
+
+from weakref import ref as wref
+
+from greenlet import getcurrent
+
+from gevent import config as GEVENT_CONFIG
+from gevent.monkey import get_original
+from gevent.events import notify
+from gevent.events import EventLoopBlocked
+from gevent.events import MemoryUsageThresholdExceeded
+from gevent.events import MemoryUsageUnderThreshold
+from gevent.events import IPeriodicMonitorThread
+from gevent.events import implementer
+
+from gevent._tracer import GreenletTracer
+from gevent._compat import thread_mod_name
+from gevent._compat import perf_counter
+
+
+
+__all__ = [
+    'PeriodicMonitoringThread',
+]
+
+get_thread_ident = get_original(thread_mod_name, 'get_ident')
+start_new_thread = get_original(thread_mod_name, 'start_new_thread')
+thread_sleep = get_original('time', 'sleep')
+
+
+
+class MonitorWarning(RuntimeWarning):
+    """The type of warnings we emit."""
+
+
+class _MonitorEntry(object):
+
+    __slots__ = ('function', 'period', 'last_run_time')
+
+    def __init__(self, function, period):
+        self.function = function
+        self.period = period
+        self.last_run_time = 0
+
+    def __eq__(self, other):
+        return self.function == other.function and self.period == other.period
+
+    def __repr__(self):
+        return repr((self.function, self.period, self.last_run_time))
+
+
+@implementer(IPeriodicMonitorThread)
+class PeriodicMonitoringThread(object):
+    # This doesn't extend threading.Thread because that gets monkey-patched.
+    # We use the low-level 'start_new_thread' primitive instead.
+
+    # The amount of seconds we will sleep when we think we have nothing
+    # to do.
+    inactive_sleep_time = 2.0
+
+    # The absolute minimum we will sleep, regardless of
+    # what particular monitoring functions want to say.
+    min_sleep_time = 0.005
+
+    # The minimum period in seconds at which we will check memory usage.
+    # Getting memory usage is fairly expensive.
+    min_memory_monitor_period = 2
+
+    # A list of _MonitorEntry objects: [(function(hub), period, last_run_time))]
+    # The first entry is always our entry for self.monitor_blocking
+    _monitoring_functions = None
+
+    # The calculated min sleep time for the monitoring functions list.
+    _calculated_sleep_time = None
+
+    # A boolean value that also happens to capture the
+    # memory usage at the time we exceeded the threshold. Reset
+    # to 0 when we go back below.
+    _memory_exceeded = 0
+
+    # The instance of GreenletTracer we're using
+    _greenlet_tracer = None
+
+    def __init__(self, hub):
+        self._hub_wref = wref(hub, self._on_hub_gc)
+        self.should_run = True
+
+        # Must be installed in the thread that the hub is running in;
+        # the trace function is threadlocal
+        assert get_thread_ident() == hub.thread_ident
+        self._greenlet_tracer = GreenletTracer()
+
+        self._monitoring_functions = [_MonitorEntry(self.monitor_blocking,
+                                                    GEVENT_CONFIG.max_blocking_time)]
+        self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
+        # Create the actual monitoring thread. This is effectively a "daemon"
+        # thread.
+        self.monitor_thread_ident = start_new_thread(self, ())
+
+        # We must track the PID to know if your thread has died after a fork
+        self.pid = os.getpid()
+
+    def _on_fork(self):
+        # Pseudo-standard method that resolver_ares and threadpool
+        # also have, called by hub.reinit()
+        pid = os.getpid()
+        if pid != self.pid:
+            self.pid = pid
+            self.monitor_thread_ident = start_new_thread(self, ())
+
+    @property
+    def hub(self):
+        return self._hub_wref()
+
+
+    def monitoring_functions(self):
+        # Return a list of _MonitorEntry objects
+
+        # Update max_blocking_time each time.
+        mbt = GEVENT_CONFIG.max_blocking_time # XXX: Events so we know when this changes.
+        if mbt != self._monitoring_functions[0].period:
+            self._monitoring_functions[0].period = mbt
+            self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
+        return self._monitoring_functions
+
+    def add_monitoring_function(self, function, period):
+        if not callable(function):
+            raise ValueError("function must be callable")
+
+        if period is None:
+            # Remove.
+            self._monitoring_functions = [
+                x for x in self._monitoring_functions
+                if x.function != function
+            ]
+        elif period <= 0:
+            raise ValueError("Period must be positive.")
+        else:
+            # Add or update period
+            entry = _MonitorEntry(function, period)
+            self._monitoring_functions = [
+                x if x.function != function else entry
+                for x in self._monitoring_functions
+            ]
+            if entry not in self._monitoring_functions:
+                self._monitoring_functions.append(entry)
+        self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
+
+    def calculate_sleep_time(self):
+        min_sleep = self._calculated_sleep_time
+        if min_sleep <= 0:
+            # Everyone wants to be disabled. Sleep for a longer period of
+            # time than usual so we don't spin unnecessarily. We might be
+            # enabled again in the future.
+            return self.inactive_sleep_time
+        return max((min_sleep, self.min_sleep_time))
+
+    def kill(self):
+        if not self.should_run:
+            # Prevent overwriting trace functions.
+            return
+        # Stop this monitoring thread from running.
+        self.should_run = False
+        # Uninstall our tracing hook
+        self._greenlet_tracer.kill()
+
+    def _on_hub_gc(self, _):
+        self.kill()
+
+    def __call__(self):
+        # The function that runs in the monitoring thread.
+        # We cannot use threading.current_thread because it would
+        # create an immortal DummyThread object.
+        getcurrent().gevent_monitoring_thread = wref(self)
+
+        try:
+            while self.should_run:
+                functions = self.monitoring_functions()
+                assert functions
+                sleep_time = self.calculate_sleep_time()
+
+                thread_sleep(sleep_time)
+
+                # Make sure the hub is still around, and still active,
+                # and keep it around while we are here.
+                hub = self.hub
+                if not hub:
+                    self.kill()
+
+                if self.should_run:
+                    this_run = perf_counter()
+                    for entry in functions:
+                        f = entry.function
+                        period = entry.period
+                        last_run = entry.last_run_time
+                        if period and last_run + period <= this_run:
+                            entry.last_run_time = this_run
+                            f(hub)
+                del hub # break our reference to hub while we sleep
+
+        except SystemExit:
+            pass
+        except: # pylint:disable=bare-except
+            # We're a daemon thread, so swallow any exceptions that get here
+            # during interpreter shutdown.
+            if not sys or not sys.stderr: # pragma: no cover
+                # Interpreter is shutting down
+                pass
+            else:
+                hub = self.hub
+                if hub is not None:
+                    # XXX: This tends to do bad things like end the process, because we
+                    # try to switch *threads*, which can't happen. Need something better.
+                    hub.handle_error(self, *sys.exc_info())
+
+    def monitor_blocking(self, hub):
+        # Called periodically to see if the trace function has
+        # fired to switch greenlets. If not, we will print
+        # the greenlet tree.
+
+        # For tests, we return a true value when we think we found something
+        # blocking
+
+        did_block = self._greenlet_tracer.did_block_hub(hub)
+        if not did_block:
+            return
+
+        active_greenlet = did_block[1]
+        report = self._greenlet_tracer.did_block_hub_report(
+            hub, active_greenlet,
+            dict(greenlet_stacks=False, current_thread_ident=self.monitor_thread_ident))
+
+        stream = hub.exception_stream
+        for line in report:
+            # Printing line by line may interleave with other things,
+            # but it should also prevent a "reentrant call to print"
+            # when the report is large.
+            print(line, file=stream)
+
+        notify(EventLoopBlocked(active_greenlet, GEVENT_CONFIG.max_blocking_time, report))
+        return (active_greenlet, report)
+
+    def ignore_current_greenlet_blocking(self):
+        self._greenlet_tracer.ignore_current_greenlet_blocking()
+
+    def monitor_current_greenlet_blocking(self):
+        self._greenlet_tracer.monitor_current_greenlet_blocking()
+
+    def _get_process(self): # pylint:disable=method-hidden
+        try:
+            # The standard library 'resource' module doesn't provide
+            # a standard way to get the RSS measure, only the maximum.
+            # You might be tempted to try to compute something by adding
+            # together text and data sizes, but on many systems those come back
+            # zero. So our only option is psutil.
+            from psutil import Process, AccessDenied
+            # Make sure it works (why would we be denied access to our own process?)
+            try:
+                proc = Process()
+                proc.memory_full_info()
+            except AccessDenied: # pragma: no cover
+                proc = None
+        except ImportError:
+            proc = None
+
+        self._get_process = lambda: proc
+        return proc
+
+    def can_monitor_memory_usage(self):
+        return self._get_process() is not None
+
+    def install_monitor_memory_usage(self):
+        # Start monitoring memory usage, if possible.
+        # If not possible, emit a warning.
+        if not self.can_monitor_memory_usage():
+            import warnings
+            warnings.warn("Unable to monitor memory usage. Install psutil.",
+                          MonitorWarning)
+            return
+
+        self.add_monitoring_function(self.monitor_memory_usage,
+                                     max(GEVENT_CONFIG.memory_monitor_period,
+                                         self.min_memory_monitor_period))
+
+    def monitor_memory_usage(self, _hub):
+        max_allowed = GEVENT_CONFIG.max_memory_usage
+        if not max_allowed:
+            # They disabled it.
+            return -1 # value for tests
+
+        rusage = self._get_process().memory_full_info()
+        # uss only documented available on Windows, Linux, and OS X.
+        # If not available, fall back to rss as an aproximation.
+        mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
+
+        event = None # Return value for tests
+
+        if mem_usage > max_allowed:
+            if mem_usage > self._memory_exceeded:
+                # We're still growing
+                event = MemoryUsageThresholdExceeded(
+                    mem_usage, max_allowed, rusage)
+                notify(event)
+            self._memory_exceeded = mem_usage
+        else:
+            # we're below. Were we above it last time?
+            if self._memory_exceeded:
+                event = MemoryUsageUnderThreshold(
+                    mem_usage, max_allowed, rusage, self._memory_exceeded)
+                notify(event)
+            self._memory_exceeded = 0
+
+        return event
+
+    def __repr__(self):
+        return '<%s at %s in thread %s greenlet %r for %r>' % (
+            self.__class__.__name__,
+            hex(id(self)),
+            hex(self.monitor_thread_ident),
+            getcurrent(),
+            self._hub_wref())

python/gevent/_patcher.py

@@ -0,0 +1,127 @@
+# Copyright 2018 gevent. See LICENSE for details.
+
+# Portions of the following are inspired by code from eventlet. I
+# believe they are distinct enough that no eventlet copyright would
+# apply (they are not a copy or substantial portion of the eventlot
+# code).
+
+# Added in gevent 1.3a2. Not public in that release.
+
+from __future__ import absolute_import, print_function
+
+import importlib
+import sys
+
+from gevent._compat import PY3
+from gevent._compat import iteritems
+from gevent._compat import imp_acquire_lock
+from gevent._compat import imp_release_lock
+
+
+from gevent.builtins import __import__ as _import
+
+
+MAPPING = {
+    'gevent.local': '_threading_local',
+    'gevent.socket': 'socket',
+    'gevent.select': 'select',
+    'gevent.ssl': 'ssl',
+    'gevent.thread': '_thread' if PY3 else 'thread',
+    'gevent.subprocess': 'subprocess',
+    'gevent.os': 'os',
+    'gevent.threading': 'threading',
+    'gevent.builtins': 'builtins' if PY3 else '__builtin__',
+    'gevent.signal': 'signal',
+    'gevent.time': 'time',
+    'gevent.queue': 'queue' if PY3 else 'Queue',
+}
+
+_PATCH_PREFIX = '__g_patched_module_'
+
+class _SysModulesPatcher(object):
+
+    def __init__(self, importing):
+        self._saved = {}
+        self.importing = importing
+        self.green_modules = {
+            stdlib_name: importlib.import_module(gevent_name)
+            for gevent_name, stdlib_name
+            in iteritems(MAPPING)
+        }
+        self.orig_imported = frozenset(sys.modules)
+
+    def _save(self):
+        for modname in self.green_modules:
+            self._saved[modname] = sys.modules.get(modname, None)
+
+        self._saved[self.importing] = sys.modules.get(self.importing, None)
+        # Anything we've already patched regains its original name during this
+        # process
+        for mod_name, mod in iteritems(sys.modules):
+            if mod_name.startswith(_PATCH_PREFIX):
+                orig_mod_name = mod_name[len(_PATCH_PREFIX):]
+                self._saved[mod_name] = sys.modules.get(orig_mod_name, None)
+                self.green_modules[orig_mod_name] = mod
+
+    def _replace(self):
+        # Cover the target modules so that when you import the module it
+        # sees only the patched versions
+        for name, mod in iteritems(self.green_modules):
+            sys.modules[name] = mod
+
+    def _restore(self):
+        for modname, mod in iteritems(self._saved):
+            if mod is not None:
+                sys.modules[modname] = mod
+            else:
+                try:
+                    del sys.modules[modname]
+                except KeyError:
+                    pass
+        # Anything from the same package tree we imported this time
+        # needs to be saved so we can restore it later, and so it doesn't
+        # leak into the namespace.
+        pkg_prefix = self.importing.split('.', 1)[0]
+        for modname, mod in list(iteritems(sys.modules)):
+            if (modname not in self.orig_imported
+                    and modname != self.importing
+                    and not modname.startswith(_PATCH_PREFIX)
+                    and modname.startswith(pkg_prefix)):
+                sys.modules[_PATCH_PREFIX + modname] = mod
+                del sys.modules[modname]
+
+    def __exit__(self, t, v, tb):
+        try:
+            self._restore()
+        finally:
+            imp_release_lock()
+
+    def __enter__(self):
+        imp_acquire_lock()
+        self._save()
+        self._replace()
+
+
+def import_patched(module_name):
+    """
+    Import *module_name* with gevent monkey-patches active,
+    and return the greened module.
+
+    Any sub-modules that were imported by the package are also
+    saved.
+
+    """
+    patched_name = _PATCH_PREFIX + module_name
+    if patched_name in sys.modules:
+        return sys.modules[patched_name]
+
+
+    # Save the current module state, and restore on exit,
+    # capturing desirable changes in the modules package.
+    with _SysModulesPatcher(module_name):
+        sys.modules.pop(module_name, None)
+
+        module = _import(module_name, {}, {}, module_name.split('.')[:-1])
+        sys.modules[patched_name] = module
+
+    return module

python/gevent/_queue.pxd

@@ -0,0 +1,74 @@
+cimport cython
+from gevent.__waiter cimport Waiter
+from gevent._event cimport Event
+
+cdef _heappush
+cdef _heappop
+cdef _heapify
+
+@cython.final
+cdef _safe_remove(deq, item)
+
+@cython.final
+@cython.internal
+cdef class ItemWaiter(Waiter):
+    cdef readonly item
+    cdef readonly queue
+
+cdef class Queue:
+    cdef __weakref__
+    cdef readonly hub
+    cdef readonly queue
+
+    cdef getters
+    cdef putters
+
+    cdef _event_unlock
+    cdef Py_ssize_t _maxsize
+
+    cpdef _get(self)
+    cpdef _put(self, item)
+    cpdef _peek(self)
+
+    cpdef Py_ssize_t qsize(self)
+    cpdef bint empty(self)
+    cpdef bint full(self)
+
+    cpdef put(self, item, block=*, timeout=*)
+    cpdef put_nowait(self, item)
+
+    cdef __get_or_peek(self, method, block, timeout)
+
+    cpdef get(self, block=*, timeout=*)
+    cpdef get_nowait(self)
+    cpdef peek(self, block=*, timeout=*)
+    cpdef peek_nowait(self)
+
+    cdef _schedule_unlock(self)
+
+@cython.final
+cdef class UnboundQueue(Queue):
+    pass
+
+cdef class PriorityQueue(Queue):
+    pass
+
+cdef class LifoQueue(Queue):
+    pass
+
+cdef class JoinableQueue(Queue):
+    cdef Event _cond
+    cdef readonly int unfinished_tasks
+
+
+cdef class Channel:
+    cdef __weakref__
+    cdef readonly getters
+    cdef readonly putters
+    cdef readonly hub
+    cdef _event_unlock
+
+    cpdef get(self, block=*, timeout=*)
+    cpdef get_nowait(self)
+
+    cdef _schedule_unlock(self)

python/gevent/_semaphore.py

@@ -1,10 +1,22 @@
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+from __future__ import print_function, absolute_import, division
 import sys
-from gevent.hub import get_hub, getcurrent
+
 from gevent.timeout import Timeout
 
 
-__all__ = ['Semaphore', 'BoundedSemaphore']
+__all__ = [
+    'Semaphore',
+    'BoundedSemaphore',
+]
 
+# In Cython, we define these as 'cdef [inline]' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+locals()['get_hub'] = __import__('gevent').get_hub
 
 class Semaphore(object):
     """
@@ -75,7 +87,8 @@ class Semaphore(object):
             # NOTE: Passing the bound method will cause a memory leak on PyPy
             # with Cython <= 0.23.3. You must use >= 0.23.4.
             # See  https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext#comment-22371546
-            self._notifier = get_hub().loop.run_callback(self._notify_links)
+            hub = get_hub() # pylint:disable=undefined-variable
+            self._notifier = hub.loop.run_callback(self._notify_links)
 
     def _notify_links(self):
         # Subclasses CANNOT override. This is a cdef method.
@@ -100,7 +113,7 @@ class Semaphore(object):
                     try:
                         link(self) # Must use Cython >= 0.23.4 on PyPy else this leaks memory
                     except: # pylint:disable=bare-except
-                        getcurrent().handle_error((link, self), *sys.exc_info())
+                        getcurrent().handle_error((link, self), *sys.exc_info()) # pylint:disable=undefined-variable
                     if self._dirty:
                         # We mutated self._links so we need to start over
                         break
@@ -157,13 +170,13 @@ class Semaphore(object):
         elapses, return the exception. Otherwise, return None.
         Raises timeout if a different timer expires.
         """
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             timer = Timeout._start_new_or_dummy(timeout)
             try:
                 try:
-                    result = get_hub().switch()
+                    result = get_hub().switch() # pylint:disable=undefined-variable
                     assert result is self, 'Invalid switch into Semaphore.wait/acquire(): %r' % (result, )
                 except Timeout as ex:
                     if ex is not timer:
@@ -266,4 +279,25 @@ class BoundedSemaphore(Semaphore):
     def release(self):
         if self.counter >= self._initial_value:
             raise self._OVER_RELEASE_ERROR("Semaphore released too many times")
-        return Semaphore.release(self)
+        Semaphore.release(self)
+
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+# By building the semaphore with Cython under PyPy, we get
+# atomic operations (specifically, exiting/releasing), at the
+# cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
+# at around 1s and the compiled version at around 4s). Some clever subclassing
+# and having only the bare minimum be in cython might help reduce that penalty.
+# NOTE: You must use version 0.23.4 or later to avoid a memory leak.
+# https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
+# However, that's all for naught on up to and including PyPy 4.0.1 which
+# have some serious crashing bugs with GC interacting with cython.
+# It hasn't been tested since then, and PURE_PYTHON is assumed to be true
+# for PyPy in all cases anyway, so this does nothing.
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__semaphore')

python/gevent/_socket2.py

@@ -2,13 +2,15 @@
 """
 Python 2 socket module.
 """
+from __future__ import absolute_import
+
 # Our import magic sadly makes this warning useless
 # pylint: disable=undefined-variable
 
-import time
 from gevent import _socketcommon
 from gevent._util import copy_globals
 from gevent._compat import PYPY
+from gevent.timeout import Timeout
 
 copy_globals(_socketcommon, globals(),
              names_to_ignore=_socketcommon.__py3_imports__ + _socketcommon.__extensions__,
@@ -36,22 +38,35 @@ else:
     # Python 2 doesn't natively support with statements on _fileobject;
     # but it eases our test cases if we can do the same with on both Py3
     # and Py2. Implementation copied from Python 3
-    if not hasattr(_fileobject, '__enter__'):
-        # we could either patch in place:
-        #_fileobject.__enter__ = lambda self: self
-        #_fileobject.__exit__ = lambda self, *args: self.close() if not self.closed else None
-        # or we could subclass. subclassing has the benefit of not
-        # changing the behaviour of the stdlib if we're just imported; OTOH,
-        # under Python 2.6/2.7, test_urllib2net.py asserts that the class IS
-        # socket._fileobject (sigh), so we have to work around that.
-        class _fileobject(_fileobject): # pylint:disable=function-redefined
+    assert not hasattr(_fileobject, '__enter__')
+    # we could either patch in place:
+    #_fileobject.__enter__ = lambda self: self
+    #_fileobject.__exit__ = lambda self, *args: self.close() if not self.closed else None
+    # or we could subclass. subclassing has the benefit of not
+    # changing the behaviour of the stdlib if we're just imported; OTOH,
+    # under Python 2.6/2.7, test_urllib2net.py asserts that the class IS
+    # socket._fileobject (sigh), so we have to work around that.
 
-            def __enter__(self):
-                return self
+    # We also make it call our custom socket closing method that disposes
+    # if IO watchers but not the actual socket itself.
+
+    # Python 2 relies on reference counting to close sockets, so this is all
+    # very ugly and fragile.
+
+    class _fileobject(_fileobject): # pylint:disable=function-redefined
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            if not self.closed:
+                self.close()
+
+        def close(self):
+            if self._sock is not None:
+                self._sock._drop_events()
+            super(_fileobject, self).close()
 
-            def __exit__(self, *args):
-                if not self.closed:
-                    self.close()
 
 def _get_memory(data):
     try:
@@ -89,6 +104,7 @@ class _closedsocket(object):
 
 timeout_default = object()
 
+from gevent._hub_primitives import wait_on_socket as _wait_on_socket
 
 class socket(object):
     """
@@ -108,11 +124,13 @@ class socket(object):
             self.timeout = _socket.getdefaulttimeout()
         else:
             if hasattr(_sock, '_sock'):
+                # passed a gevent socket
                 self._sock = _sock._sock
                 self.timeout = getattr(_sock, 'timeout', False)
                 if self.timeout is False:
                     self.timeout = _socket.getdefaulttimeout()
             else:
+                # passed a native socket
                 self._sock = _sock
                 self.timeout = _socket.getdefaulttimeout()
             if PYPY:
@@ -164,25 +182,7 @@ class socket(object):
 
     ref = property(_get_ref, _set_ref)
 
-    def _wait(self, watcher, timeout_exc=timeout('timed out')):
-        """Block the current greenlet until *watcher* has pending events.
-
-        If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
-        By default *timeout_exc* is ``socket.timeout('timed out')``.
-
-        If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
-        """
-        if watcher.callback is not None:
-            raise _socketcommon.ConcurrentObjectUseError('This socket is already used by another greenlet: %r' % (watcher.callback, ))
-        if self.timeout is not None:
-            timeout = Timeout.start_new(self.timeout, timeout_exc, ref=False)
-        else:
-            timeout = None
-        try:
-            self.hub.wait(watcher)
-        finally:
-            if timeout is not None:
-                timeout.cancel()
+    _wait = _wait_on_socket
 
     def accept(self):
         sock = self._sock
@@ -200,10 +200,21 @@ class socket(object):
             client_socket._drop()
         return sockobj, address
 
-    def close(self, _closedsocket=_closedsocket, cancel_wait_ex=cancel_wait_ex):
+    def _drop_events(self, cancel_wait_ex=cancel_wait_ex):
+        if self._read_event is not None:
+            self.hub.cancel_wait(self._read_event, cancel_wait_ex, True)
+            self._read_event = None
+        if self._write_event is not None:
+            self.hub.cancel_wait(self._write_event, cancel_wait_ex, True)
+            self._write_event = None
+
+
+    def close(self, _closedsocket=_closedsocket):
         # This function should not reference any globals. See Python issue #808164.
-        self.hub.cancel_wait(self._read_event, cancel_wait_ex)
-        self.hub.cancel_wait(self._write_event, cancel_wait_ex)
+
+        # Also break any reference to the loop.io objects. Our fileno, which they were
+        # tied to, is now free to be reused, so these objects are no longer functional.
+        self._drop_events()
         s = self._sock
         self._sock = _closedsocket()
         if PYPY:
@@ -217,13 +228,9 @@ class socket(object):
         if self.timeout == 0.0:
             return self._sock.connect(address)
         sock = self._sock
-        if isinstance(address, tuple):
-            r = getaddrinfo(address[0], address[1], sock.family)
-            address = r[0][-1]
-        if self.timeout is not None:
-            timer = Timeout.start_new(self.timeout, timeout('timed out'))
-        else:
-            timer = None
+        address = _socketcommon._resolve_addr(sock, address)
+
+        timer = Timeout._start_new_or_dummy(self.timeout, timeout('timed out'))
         try:
             while True:
                 err = sock.getsockopt(SOL_SOCKET, SO_ERROR)
@@ -237,8 +244,7 @@ class socket(object):
                 else:
                     raise error(result, strerror(result))
         finally:
-            if timer is not None:
-                timer.cancel()
+            timer.close()
 
     def connect_ex(self, address):
         try:
@@ -248,8 +254,7 @@ class socket(object):
         except error as ex:
             if type(ex) is error: # pylint:disable=unidiomatic-typecheck
                 return ex.args[0]
-            else:
-                raise  # gaierror is not silenced by connect_ex
+            raise  # gaierror is not silenced by connect_ex
 
     def dup(self):
         """dup() -> socket object
@@ -261,7 +266,14 @@ class socket(object):
     def makefile(self, mode='r', bufsize=-1):
         # Two things to look out for:
         # 1) Closing the original socket object should not close the
-        #    socket (hence creating a new instance)
+        #    fileobject (hence creating a new socket instance);
+        #    An alternate approach is what _socket3.py does, which is to
+        #    keep count of the times makefile objects have been opened (Py3's
+        #    SocketIO helps with that). But the newly created socket, which
+        #    has its own read/write watchers, does need those to be closed
+        #    when the fileobject is; our custom subclass does that. Note that
+        #    we can't pass the 'close=True' argument, as that causes reference counts
+        #    to get screwed up, and Python2 sockets rely on those.
         # 2) The resulting fileobject must keep the timeout in order
         #    to be compatible with the stdlib's socket.makefile.
         # Pass self as _sock to preserve timeout.
@@ -322,7 +334,7 @@ class socket(object):
         try:
             return sock.send(data, flags)
         except error as ex:
-            if ex.args[0] != EWOULDBLOCK or timeout == 0.0:
+            if ex.args[0] not in _socketcommon.GSENDAGAIN or timeout == 0.0:
                 raise
             sys.exc_clear()
             self._wait(self._write_event)
@@ -333,91 +345,13 @@ class socket(object):
                     return 0
                 raise
 
-    def __send_chunk(self, data_memory, flags, timeleft, end):
-        """
-        Send the complete contents of ``data_memory`` before returning.
-        This is the core loop around :meth:`send`.
-
-        :param timeleft: Either ``None`` if there is no timeout involved,
-           or a float indicating the timeout to use.
-        :param end: Either ``None`` if there is no timeout involved, or
-           a float giving the absolute end time.
-        :return: An updated value for ``timeleft`` (or None)
-        :raises timeout: If ``timeleft`` was given and elapsed while
-           sending this chunk.
-        """
-        data_sent = 0
-        len_data_memory = len(data_memory)
-        started_timer = 0
-        while data_sent < len_data_memory:
-            chunk = data_memory[data_sent:]
-            if timeleft is None:
-                data_sent += self.send(chunk, flags)
-            elif started_timer and timeleft <= 0:
-                # Check before sending to guarantee a check
-                # happens even if each chunk successfully sends its data
-                # (especially important for SSL sockets since they have large
-                # buffers). But only do this if we've actually tried to
-                # send something once to avoid spurious timeouts on non-blocking
-                # sockets.
-                raise timeout('timed out')
-            else:
-                started_timer = 1
-                data_sent += self.send(chunk, flags, timeout=timeleft)
-                timeleft = end - time.time()
-
-        return timeleft
-
     def sendall(self, data, flags=0):
         if isinstance(data, unicode):
             data = data.encode()
         # this sendall is also reused by gevent.ssl.SSLSocket subclass,
         # so it should not call self._sock methods directly
         data_memory = _get_memory(data)
-        len_data_memory = len(data_memory)
-        if not len_data_memory:
-            # Don't send empty data, can cause SSL EOFError.
-            # See issue 719
-            return 0
-
-        # On PyPy up through 2.6.0, subviews of a memoryview() object
-        # copy the underlying bytes the first time the builtin
-        # socket.send() method is called. On a non-blocking socket
-        # (that thus calls socket.send() many times) with a large
-        # input, this results in many repeated copies of an ever
-        # smaller string, depending on the networking buffering. For
-        # example, if each send() can process 1MB of a 50MB input, and
-        # we naively pass the entire remaining subview each time, we'd
-        # copy 49MB, 48MB, 47MB, etc, thus completely killing
-        # performance. To workaround this problem, we work in
-        # reasonable, fixed-size chunks. This results in a 10x
-        # improvement to bench_sendall.py, while having no measurable impact on
-        # CPython (since it doesn't copy at all the only extra overhead is
-        # a few python function calls, which is negligible for large inputs).
-
-        # See https://bitbucket.org/pypy/pypy/issues/2091/non-blocking-socketsend-slow-gevent
-
-        # Too small of a chunk (the socket's buf size is usually too
-        # small) results in reduced perf due to *too many* calls to send and too many
-        # small copies. With a buffer of 143K (the default on my system), for
-        # example, bench_sendall.py yields ~264MB/s, while using 1MB yields
-        # ~653MB/s (matching CPython). 1MB is arbitrary and might be better
-        # chosen, say, to match a page size?
-        chunk_size = max(self.getsockopt(SOL_SOCKET, SO_SNDBUF), 1024 * 1024) # pylint:disable=no-member
-
-        data_sent = 0
-        end = None
-        timeleft = None
-        if self.timeout is not None:
-            timeleft = self.timeout
-            end = time.time() + timeleft
-
-        while data_sent < len_data_memory:
-            chunk_end = min(data_sent + chunk_size, len_data_memory)
-            chunk = data_memory[data_sent:chunk_end]
-
-            timeleft = self.__send_chunk(chunk, flags, timeleft, end)
-            data_sent += len(chunk) # Guaranteed it sent the whole thing
+        return _socketcommon._sendall(self, data_memory, flags)
 
     def sendto(self, *args):
         sock = self._sock

python/gevent/_socket3.py

@@ -10,17 +10,27 @@ from __future__ import absolute_import
 import io
 import os
 import sys
-import time
+
 from gevent import _socketcommon
 from gevent._util import copy_globals
 from gevent._compat import PYPY
+from gevent.timeout import Timeout
 import _socket
 from os import dup
 
+
 copy_globals(_socketcommon, globals(),
              names_to_ignore=_socketcommon.__extensions__,
              dunder_names_to_keep=())
 
+try:
+    from errno import EHOSTUNREACH
+    from errno import ECONNREFUSED
+except ImportError:
+    EHOSTUNREACH = -1
+    ECONNREFUSED = -1
+
+
 __socket__ = _socketcommon.__socket__
 __implements__ = _socketcommon._implements
 __extensions__ = _socketcommon.__extensions__
@@ -58,6 +68,7 @@ class _wrefsocket(_socket.socket):
         timeout = property(lambda s: s.gettimeout(),
                            lambda s, nv: s.settimeout(nv))
 
+from gevent._hub_primitives import wait_on_socket as _wait_on_socket
 
 class socket(object):
     """
@@ -74,13 +85,37 @@ class socket(object):
     # of _wrefsocket. (gevent internal usage only)
     _gevent_sock_class = _wrefsocket
 
-    def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None):
-        # Take the same approach as socket2: wrap a real socket object,
-        # don't subclass it. This lets code that needs the raw _sock (not tied to the hub)
-        # get it. This shows up in tests like test__example_udp_server.
-        self._sock = self._gevent_sock_class(family, type, proto, fileno)
-        self._io_refs = 0
-        self._closed = False
+    _io_refs = 0
+    _closed = False
+    _read_event = None
+    _write_event = None
+
+
+    # Take the same approach as socket2: wrap a real socket object,
+    # don't subclass it. This lets code that needs the raw _sock (not tied to the hub)
+    # get it. This shows up in tests like test__example_udp_server.
+
+    if sys.version_info[:2] < (3, 7):
+        def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None):
+            self._sock = self._gevent_sock_class(family, type, proto, fileno)
+            self.timeout = None
+            self.__init_common()
+    else:
+        # In 3.7, socket changed to auto-detecting family, type, and proto
+        # when given a fileno.
+        def __init__(self, family=-1, type=-1, proto=-1, fileno=None):
+            if fileno is None:
+                if family == -1:
+                    family = AF_INET
+                if type == -1:
+                    type = SOCK_STREAM
+                if proto == -1:
+                    proto = 0
+            self._sock = self._gevent_sock_class(family, type, proto, fileno)
+            self.timeout = None
+            self.__init_common()
+
+    def __init_common(self):
         _socket.socket.setblocking(self._sock, False)
         fileno = _socket.socket.fileno(self._sock)
         self.hub = get_hub()
@@ -101,6 +136,16 @@ class socket(object):
                 return self._sock.type & ~_socket.SOCK_NONBLOCK # pylint:disable=no-member
             return self._sock.type
 
+    def getblocking(self):
+        """
+        Returns whether the socket will approximate blocking
+        behaviour.
+
+        .. versionadded:: 1.3a2
+            Added in Python 3.7.
+        """
+        return self.timeout != 0.0
+
     def __enter__(self):
         return self
 
@@ -114,7 +159,7 @@ class socket(object):
             s = _socket.socket.__repr__(self._sock)
         except Exception as ex: # pylint:disable=broad-except
             # Observed on Windows Py3.3, printing the repr of a socket
-            # that just sufferred a ConnectionResetError [WinError 10054]:
+            # that just suffered a ConnectionResetError [WinError 10054]:
             # "OverflowError: no printf formatter to display the socket descriptor in decimal"
             # Not sure what the actual cause is or if there's a better way to handle this
             s = '<socket [%r]>' % ex
@@ -138,25 +183,7 @@ class socket(object):
 
     ref = property(_get_ref, _set_ref)
 
-    def _wait(self, watcher, timeout_exc=timeout('timed out')):
-        """Block the current greenlet until *watcher* has pending events.
-
-        If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
-        By default *timeout_exc* is ``socket.timeout('timed out')``.
-
-        If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
-        """
-        if watcher.callback is not None:
-            raise _socketcommon.ConcurrentObjectUseError('This socket is already used by another greenlet: %r' % (watcher.callback, ))
-        if self.timeout is not None:
-            timeout = Timeout.start_new(self.timeout, timeout_exc, ref=False)
-        else:
-            timeout = None
-        try:
-            self.hub.wait(watcher)
-        finally:
-            if timeout is not None:
-                timeout.cancel()
+    _wait = _wait_on_socket
 
     def dup(self):
         """dup() -> socket object
@@ -237,15 +264,28 @@ class socket(object):
         return text
 
     def _decref_socketios(self):
+        # Called by SocketIO when it is closed.
         if self._io_refs > 0:
             self._io_refs -= 1
         if self._closed:
             self.close()
 
+    def _drop_events(self):
+        if self._read_event is not None:
+            self.hub.cancel_wait(self._read_event, cancel_wait_ex, True)
+            self._read_event = None
+        if self._write_event is not None:
+            self.hub.cancel_wait(self._write_event, cancel_wait_ex, True)
+            self._write_event = None
+
     def _real_close(self, _ss=_socket.socket, cancel_wait_ex=cancel_wait_ex):
         # This function should not reference any globals. See Python issue #808164.
-        self.hub.cancel_wait(self._read_event, cancel_wait_ex)
-        self.hub.cancel_wait(self._write_event, cancel_wait_ex)
+
+        # Break any reference to the loop.io objects. Our fileno,
+        # which they were tied to, is now free to be reused, so these
+        # objects are no longer functional.
+        self._drop_events()
+
         _ss.close(self._sock)
 
         # Break any references to the underlying socket object. Tested
@@ -286,35 +326,41 @@ class socket(object):
     def connect(self, address):
         if self.timeout == 0.0:
             return _socket.socket.connect(self._sock, address)
-        if isinstance(address, tuple):
-            r = getaddrinfo(address[0], address[1], self.family)
-            address = r[0][-1]
-        if self.timeout is not None:
-            timer = Timeout.start_new(self.timeout, timeout('timed out'))
-        else:
-            timer = None
-        try:
+        address = _socketcommon._resolve_addr(self._sock, address)
+
+        with Timeout._start_new_or_dummy(self.timeout, timeout("timed out")):
             while True:
                 err = self.getsockopt(SOL_SOCKET, SO_ERROR)
                 if err:
                     raise error(err, strerror(err))
                 result = _socket.socket.connect_ex(self._sock, address)
+
                 if not result or result == EISCONN:
                     break
                 elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
                     self._wait(self._write_event)
                 else:
+                    if (isinstance(address, tuple)
+                            and address[0] == 'fe80::1'
+                            and result == EHOSTUNREACH):
+                        # On Python 3.7 on mac, we see EHOSTUNREACH
+                        # returned for this link-local address, but it really is
+                        # supposed to be ECONNREFUSED according to the standard library
+                        # tests (test_socket.NetworkConnectionNoServer.test_create_connection)
+                        # (On previous versions, that code passed the '127.0.0.1' IPv4 address, so
+                        # ipv6 link locals were never a factor; 3.7 passes 'localhost'.)
+                        # It is something of a mystery how the stdlib socket code doesn't
+                        # produce EHOSTUNREACH---I (JAM) can't see how socketmodule.c would avoid
+                        # that. The normal connect just calls connect_ex much like we do.
+                        result = ECONNREFUSED
                     raise error(result, strerror(result))
-        finally:
-            if timer is not None:
-                timer.cancel()
 
     def connect_ex(self, address):
         try:
             return self.connect(address) or 0
         except timeout:
             return EAGAIN
-        except gaierror:
+        except gaierror: # pylint:disable=try-except-raise
             # gaierror/overflowerror/typerror is not silenced by connect_ex;
             # gaierror extends OSError (aka error) so catch it first
             raise
@@ -335,8 +381,9 @@ class socket(object):
                     raise
             self._wait(self._read_event)
 
-    if hasattr(_socket.socket, 'sendmsg'):
-        # Only on Unix
+    if hasattr(_socket.socket, 'recvmsg'):
+        # Only on Unix; PyPy 3.5 5.10.0 provides sendmsg and recvmsg, but not
+        # recvmsg_into (at least on os x)
 
         def recvmsg(self, *args):
             while True:
@@ -347,6 +394,8 @@ class socket(object):
                         raise
                 self._wait(self._read_event)
 
+    if hasattr(_socket.socket, 'recvmsg_into'):
+
         def recvmsg_into(self, *args):
             while True:
                 try:
@@ -389,7 +438,7 @@ class socket(object):
         try:
             return _socket.socket.send(self._sock, data, flags)
         except error as ex:
-            if ex.args[0] != EWOULDBLOCK or timeout == 0.0:
+            if ex.args[0] not in _socketcommon.GSENDAGAIN or timeout == 0.0:
                 raise
             self._wait(self._write_event)
             try:
@@ -406,27 +455,7 @@ class socket(object):
         # PyPy2, so it's possibly premature to do this. However, there is a 3.5 test case that
         # possibly exposes this in a severe way.
         data_memory = _get_memory(data)
-        len_data_memory = len(data_memory)
-        if not len_data_memory:
-            # Don't try to send empty data at all, no point, and breaks ssl
-            # See issue 719
-            return 0
-
-        if self.timeout is None:
-            data_sent = 0
-            while data_sent < len_data_memory:
-                data_sent += self.send(data_memory[data_sent:], flags)
-        else:
-            timeleft = self.timeout
-            end = time.time() + timeleft
-            data_sent = 0
-            while True:
-                data_sent += self.send(data_memory[data_sent:], flags, timeout=timeleft)
-                if data_sent >= len_data_memory:
-                    break
-                timeleft = end - time.time()
-                if timeleft <= 0:
-                    raise timeout('timed out')
+        return _socketcommon._sendall(self, data_memory, flags)
 
     def sendto(self, *args):
         try:
@@ -464,6 +493,11 @@ class socket(object):
                     raise
 
     def setblocking(self, flag):
+        # Beginning in 3.6.0b3 this is supposed to raise
+        # if the file descriptor is closed, but the test for it
+        # involves closing the fileno directly. Since we
+        # don't touch the fileno here, it doesn't make sense for
+        # us.
         if flag:
             self.timeout = None
         else:
@@ -659,7 +693,7 @@ if hasattr(_socket, "socketpair"):
         b = socket(family, type, proto, b.detach())
         return a, b
 
-else:
+else: # pragma: no cover
     # Origin: https://gist.github.com/4325783, by Geert Jansen.  Public domain.
 
     # gevent: taken from 3.6 release. Expected to be used only on Win. Added to Win/3.5
@@ -715,351 +749,8 @@ else:
             __implements__.remove('socketpair')
 
 
-# PyPy needs drop and reuse
-def _do_reuse_or_drop(sock, methname):
-    try:
-        method = getattr(sock, methname)
-    except (AttributeError, TypeError):
-        pass
-    else:
-        method()
-
-from io import BytesIO
-
-
-class _basefileobject(object):
-    """Faux file object attached to a socket object."""
-
-    default_bufsize = 8192
-    name = "<socket>"
-
-    __slots__ = ["mode", "bufsize", "softspace",
-                 # "closed" is a property, see below
-                 "_sock", "_rbufsize", "_wbufsize", "_rbuf", "_wbuf", "_wbuf_len",
-                 "_close"]
-
-    def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-        _do_reuse_or_drop(sock, '_reuse')
-        self._sock = sock
-        self.mode = mode # Not actually used in this version
-        if bufsize < 0:
-            bufsize = self.default_bufsize
-        self.bufsize = bufsize
-        self.softspace = False
-        # _rbufsize is the suggested recv buffer size.  It is *strictly*
-        # obeyed within readline() for recv calls.  If it is larger than
-        # default_bufsize it will be used for recv calls within read().
-        if bufsize == 0:
-            self._rbufsize = 1
-        elif bufsize == 1:
-            self._rbufsize = self.default_bufsize
-        else:
-            self._rbufsize = bufsize
-        self._wbufsize = bufsize
-        # We use BytesIO for the read buffer to avoid holding a list
-        # of variously sized string objects which have been known to
-        # fragment the heap due to how they are malloc()ed and often
-        # realloc()ed down much smaller than their original allocation.
-        self._rbuf = BytesIO()
-        self._wbuf = [] # A list of strings
-        self._wbuf_len = 0
-        self._close = close
-
-    def _getclosed(self):
-        return self._sock is None
-    closed = property(_getclosed, doc="True if the file is closed")
-
-    def close(self):
-        try:
-            if self._sock:
-                self.flush()
-        finally:
-            s = self._sock
-            self._sock = None
-            if s is not None:
-                if self._close:
-                    s.close()
-                else:
-                    _do_reuse_or_drop(s, '_drop')
-
-    def __del__(self):
-        try:
-            self.close()
-        except: # pylint:disable=bare-except
-            # close() may fail if __init__ didn't complete
-            pass
-
-    def flush(self):
-        if self._wbuf:
-            data = b"".join(self._wbuf)
-            self._wbuf = []
-            self._wbuf_len = 0
-            buffer_size = max(self._rbufsize, self.default_bufsize)
-            data_size = len(data)
-            write_offset = 0
-            view = memoryview(data)
-            try:
-                while write_offset < data_size:
-                    self._sock.sendall(view[write_offset:write_offset + buffer_size])
-                    write_offset += buffer_size
-            finally:
-                if write_offset < data_size:
-                    remainder = data[write_offset:]
-                    del view, data  # explicit free
-                    self._wbuf.append(remainder)
-                    self._wbuf_len = len(remainder)
-
-    def fileno(self):
-        return self._sock.fileno()
-
-    def write(self, data):
-        if not isinstance(data, bytes):
-            raise TypeError("Non-bytes data")
-        if not data:
-            return
-        self._wbuf.append(data)
-        self._wbuf_len += len(data)
-        if (self._wbufsize == 0 or (self._wbufsize == 1 and b'\n' in data) or
-                (self._wbufsize > 1 and self._wbuf_len >= self._wbufsize)):
-            self.flush()
-
-    def writelines(self, list):
-        # XXX We could do better here for very long lists
-        # XXX Should really reject non-string non-buffers
-        lines = filter(None, map(str, list))
-        self._wbuf_len += sum(map(len, lines))
-        self._wbuf.extend(lines)
-        if self._wbufsize <= 1 or self._wbuf_len >= self._wbufsize:
-            self.flush()
-
-    def read(self, size=-1):
-        # Use max, disallow tiny reads in a loop as they are very inefficient.
-        # We never leave read() with any leftover data from a new recv() call
-        # in our internal buffer.
-        rbufsize = max(self._rbufsize, self.default_bufsize)
-        # Our use of BytesIO rather than lists of string objects returned by
-        # recv() minimizes memory usage and fragmentation that occurs when
-        # rbufsize is large compared to the typical return value of recv().
-        buf = self._rbuf
-        buf.seek(0, 2)  # seek end
-        if size < 0:
-            # Read until EOF
-            self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
-            while True:
-                try:
-                    data = self._sock.recv(rbufsize)
-                except InterruptedError:
-                    continue
-                if not data:
-                    break
-                buf.write(data)
-            return buf.getvalue()
-        else:
-            # Read until size bytes or EOF seen, whichever comes first
-            buf_len = buf.tell()
-            if buf_len >= size:
-                # Already have size bytes in our buffer?  Extract and return.
-                buf.seek(0)
-                rv = buf.read(size)
-                self._rbuf = BytesIO()
-                self._rbuf.write(buf.read())
-                return rv
-
-            self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
-            while True:
-                left = size - buf_len
-                # recv() will malloc the amount of memory given as its
-                # parameter even though it often returns much less data
-                # than that.  The returned data string is short lived
-                # as we copy it into a BytesIO and free it.  This avoids
-                # fragmentation issues on many platforms.
-                try:
-                    data = self._sock.recv(left)
-                except InterruptedError:
-                    continue
-
-                if not data:
-                    break
-                n = len(data)
-                if n == size and not buf_len:
-                    # Shortcut.  Avoid buffer data copies when:
-                    # - We have no data in our buffer.
-                    # AND
-                    # - Our call to recv returned exactly the
-                    #   number of bytes we were asked to read.
-                    return data
-                if n == left:
-                    buf.write(data)
-                    del data  # explicit free
-                    break
-                assert n <= left, "recv(%d) returned %d bytes" % (left, n)
-                buf.write(data)
-                buf_len += n
-                del data  # explicit free
-                #assert buf_len == buf.tell()
-            return buf.getvalue()
-
-    def readline(self, size=-1):
-        # pylint:disable=too-many-return-statements
-        buf = self._rbuf
-        buf.seek(0, 2)  # seek end
-        if buf.tell() > 0:
-            # check if we already have it in our buffer
-            buf.seek(0)
-            bline = buf.readline(size)
-            if bline.endswith(b'\n') or len(bline) == size:
-                self._rbuf = BytesIO()
-                self._rbuf.write(buf.read())
-                return bline
-            del bline
-        if size < 0: # pylint:disable=too-many-nested-blocks
-            # Read until \n or EOF, whichever comes first
-            if self._rbufsize <= 1:
-                # Speed up unbuffered case
-                buf.seek(0)
-                buffers = [buf.read()]
-                self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
-                data = None
-                recv = self._sock.recv
-                while True:
-                    try:
-                        while data != b"\n":
-                            data = recv(1)
-                            if not data:
-                                break
-                            buffers.append(data)
-                    except InterruptedError:
-                        # The try..except to catch EINTR was moved outside the
-                        # recv loop to avoid the per byte overhead.
-                        continue
-
-                    break
-                return b"".join(buffers)
-
-            buf.seek(0, 2)  # seek end
-            self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
-            while True:
-                try:
-                    data = self._sock.recv(self._rbufsize)
-                except InterruptedError:
-                    continue
-
-                if not data:
-                    break
-                nl = data.find(b'\n')
-                if nl >= 0:
-                    nl += 1
-                    buf.write(data[:nl])
-                    self._rbuf.write(data[nl:])
-                    del data
-                    break
-                buf.write(data)
-            return buf.getvalue()
-        else:
-            # Read until size bytes or \n or EOF seen, whichever comes first
-            buf.seek(0, 2)  # seek end
-            buf_len = buf.tell()
-            if buf_len >= size:
-                buf.seek(0)
-                rv = buf.read(size)
-                self._rbuf = BytesIO()
-                self._rbuf.write(buf.read())
-                return rv
-            self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
-            while True:
-                try:
-                    data = self._sock.recv(self._rbufsize)
-                except InterruptedError:
-                    continue
-
-                if not data:
-                    break
-                left = size - buf_len
-                # did we just receive a newline?
-                nl = data.find(b'\n', 0, left)
-                if nl >= 0:
-                    nl += 1
-                    # save the excess data to _rbuf
-                    self._rbuf.write(data[nl:])
-                    if buf_len:
-                        buf.write(data[:nl])
-                        break
-                    else:
-                        # Shortcut.  Avoid data copy through buf when returning
-                        # a substring of our first recv().
-                        return data[:nl]
-                n = len(data)
-                if n == size and not buf_len:
-                    # Shortcut.  Avoid data copy through buf when
-                    # returning exactly all of our first recv().
-                    return data
-                if n >= left:
-                    buf.write(data[:left])
-                    self._rbuf.write(data[left:])
-                    break
-                buf.write(data)
-                buf_len += n
-                #assert buf_len == buf.tell()
-            return buf.getvalue()
-
-    def readlines(self, sizehint=0):
-        total = 0
-        list = []
-        while True:
-            line = self.readline()
-            if not line:
-                break
-            list.append(line)
-            total += len(line)
-            if sizehint and total >= sizehint:
-                break
-        return list
-
-    # Iterator protocols
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        line = self.readline()
-        if not line:
-            raise StopIteration
-        return line
-    __next__ = next
-
-try:
-    from gevent.fileobject import FileObjectPosix
-except ImportError:
-    # Manual implementation
-    _fileobject = _basefileobject
-else:
-    class _fileobject(FileObjectPosix):
-        # Add the proper drop/reuse support for pypy, and match
-        # the close=False default in the constructor
-        def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-            _do_reuse_or_drop(sock, '_reuse')
-            self._sock = sock
-            FileObjectPosix.__init__(self, sock, mode, bufsize, close)
-
-        def close(self):
-            try:
-                if self._sock:
-                    self.flush()
-            finally:
-                s = self._sock
-                self._sock = None
-                if s is not None:
-                    if self._close:
-                        FileObjectPosix.close(self)
-                    else:
-                        _do_reuse_or_drop(s, '_drop')
-
-        def __del__(self):
-            try:
-                self.close()
-            except: # pylint:disable=bare-except
-                # close() may fail if __init__ didn't complete
-                pass
-
+if hasattr(__socket__, 'close'): # Python 3.7b1+
+    close = __socket__.close # pylint:disable=no-member
+    __imports__ += ['close']
 
 __all__ = __implements__ + __extensions__ + __imports__

python/gevent/_socketcommon.py

@@ -68,16 +68,15 @@ __py3_imports__ = [
 
 __imports__.extend(__py3_imports__)
 
-
+import time
 import sys
-from gevent.hub import get_hub
-from gevent.hub import ConcurrentObjectUseError
-from gevent.timeout import Timeout
+from gevent._hub_local import get_hub_noargs as get_hub
 from gevent._compat import string_types, integer_types, PY3
 from gevent._util import copy_globals
-from gevent._util import _NONE
 
 is_windows = sys.platform == 'win32'
+is_macos = sys.platform == 'darwin'
+
 # pylint:disable=no-name-in-module,unused-import
 if is_windows:
     # no such thing as WSAEPERM or error code 10001 according to winsock.h or MSDN
@@ -102,6 +101,17 @@ try:
 except ImportError:
     EBADF = 9
 
+# macOS can return EPROTOTYPE when writing to a socket that is shutting
+# Down. Retrying the write should return the expected EPIPE error.
+# Downstream classes (like pywsgi) know how to handle/ignore EPIPE.
+# This set is used by socket.send() to decide whether the write should
+# be retried. The default is to retry only on EWOULDBLOCK. Here we add
+# EPROTOTYPE on macOS to handle this platform-specific race condition.
+GSENDAGAIN = (EWOULDBLOCK,)
+if is_macos:
+    from errno import EPROTOTYPE
+    GSENDAGAIN += (EPROTOTYPE,)
+
 import _socket
 _realsocket = _socket.socket
 import socket as __socket__
@@ -121,87 +131,13 @@ del _name, _value
 
 _timeout_error = timeout # pylint: disable=undefined-variable
 
+from gevent import _hub_primitives
+_hub_primitives.set_default_timeout_error(_timeout_error)
 
-def wait(io, timeout=None, timeout_exc=_NONE):
-    """
-    Block the current greenlet until *io* is ready.
-
-    If *timeout* is non-negative, then *timeout_exc* is raised after
-    *timeout* second has passed. By default *timeout_exc* is
-    ``socket.timeout('timed out')``.
-
-    If :func:`cancel_wait` is called on *io* by another greenlet,
-    raise an exception in this blocking greenlet
-    (``socket.error(EBADF, 'File descriptor was closed in another
-    greenlet')`` by default).
-
-    :param io: A libev watcher, most commonly an IO watcher obtained from
-        :meth:`gevent.core.loop.io`
-    :keyword timeout_exc: The exception to raise if the timeout expires.
-        By default, a :class:`socket.timeout` exception is raised.
-        If you pass a value for this keyword, it is interpreted as for
-        :class:`gevent.timeout.Timeout`.
-    """
-    if io.callback is not None:
-        raise ConcurrentObjectUseError('This socket is already used by another greenlet: %r' % (io.callback, ))
-    if timeout is not None:
-        timeout_exc = timeout_exc if timeout_exc is not _NONE else _timeout_error('timed out')
-        timeout = Timeout.start_new(timeout, timeout_exc)
-
-    try:
-        return get_hub().wait(io)
-    finally:
-        if timeout is not None:
-            timeout.cancel()
-    # rename "io" to "watcher" because wait() works with any watcher
-
-
-def wait_read(fileno, timeout=None, timeout_exc=_NONE):
-    """
-    Block the current greenlet until *fileno* is ready to read.
-
-    For the meaning of the other parameters and possible exceptions,
-    see :func:`wait`.
-
-    .. seealso:: :func:`cancel_wait`
-     """
-    io = get_hub().loop.io(fileno, 1)
-    return wait(io, timeout, timeout_exc)
-
-
-def wait_write(fileno, timeout=None, timeout_exc=_NONE, event=_NONE):
-    """
-    Block the current greenlet until *fileno* is ready to write.
-
-    For the meaning of the other parameters and possible exceptions,
-    see :func:`wait`.
-
-    :keyword event: Ignored. Applications should not pass this parameter.
-       In the future, it may become an error.
-
-    .. seealso:: :func:`cancel_wait`
-    """
-    # pylint:disable=unused-argument
-    io = get_hub().loop.io(fileno, 2)
-    return wait(io, timeout, timeout_exc)
-
-
-def wait_readwrite(fileno, timeout=None, timeout_exc=_NONE, event=_NONE):
-    """
-    Block the current greenlet until *fileno* is ready to read or
-    write.
-
-    For the meaning of the other parameters and possible exceptions,
-    see :func:`wait`.
-
-    :keyword event: Ignored. Applications should not pass this parameter.
-       In the future, it may become an error.
-
-    .. seealso:: :func:`cancel_wait`
-    """
-    # pylint:disable=unused-argument
-    io = get_hub().loop.io(fileno, 3)
-    return wait(io, timeout, timeout_exc)
+wait = _hub_primitives.wait_on_watcher
+wait_read = _hub_primitives.wait_read
+wait_write = _hub_primitives.wait_write
+wait_readwrite = _hub_primitives.wait_readwrite
 
 #: The exception raised by default on a call to :func:`cancel_wait`
 class cancel_wait_ex(error): # pylint: disable=undefined-variable
@@ -216,29 +152,13 @@ def cancel_wait(watcher, error=cancel_wait_ex):
     get_hub().cancel_wait(watcher, error)
 
 
-class BlockingResolver(object):
-
-    def __init__(self, hub=None):
-        pass
-
-    def close(self):
-        pass
-
-    for method in ['gethostbyname',
-                   'gethostbyname_ex',
-                   'getaddrinfo',
-                   'gethostbyaddr',
-                   'getnameinfo']:
-        locals()[method] = staticmethod(getattr(_socket, method))
-
-
 def gethostbyname(hostname):
     """
     gethostbyname(host) -> address
 
     Return the IP address (a string of the form '255.255.255.255') for a host.
 
-    .. seealso:: :doc:`dns`
+    .. seealso:: :doc:`/dns`
     """
     return get_hub().resolver.gethostbyname(hostname)
 
@@ -251,7 +171,7 @@ def gethostbyname_ex(hostname):
     for a host.  The host argument is a string giving a host name or IP number.
     Resolve host and port into list of address info entries.
 
-    .. seealso:: :doc:`dns`
+    .. seealso:: :doc:`/dns`
     """
     return get_hub().resolver.gethostbyname_ex(hostname)
 
@@ -271,7 +191,7 @@ def getaddrinfo(host, port, family=0, socktype=0, proto=0, flags=0):
     narrow the list of addresses returned. Passing zero as a value for each of
     these arguments selects the full range of results.
 
-    .. seealso:: :doc:`dns`
+    .. seealso:: :doc:`/dns`
     """
     return get_hub().resolver.getaddrinfo(host, port, family, socktype, proto, flags)
 
@@ -297,7 +217,7 @@ def gethostbyaddr(ip_address):
     Return the true host name, a list of aliases, and a list of IP addresses,
     for a host.  The host argument is a string giving a host name or IP number.
 
-    .. seealso:: :doc:`dns`
+    .. seealso:: :doc:`/dns`
     """
     return get_hub().resolver.gethostbyaddr(ip_address)
 
@@ -308,7 +228,7 @@ def getnameinfo(sockaddr, flags):
 
     Get host and port for a sockaddr.
 
-    .. seealso:: :doc:`dns`
+    .. seealso:: :doc:`/dns`
     """
     return get_hub().resolver.getnameinfo(sockaddr, flags)
 
@@ -341,3 +261,122 @@ def getfqdn(name=''):
         else:
             name = hostname
     return name
+
+def __send_chunk(socket, data_memory, flags, timeleft, end, timeout=_timeout_error):
+    """
+    Send the complete contents of ``data_memory`` before returning.
+    This is the core loop around :meth:`send`.
+
+    :param timeleft: Either ``None`` if there is no timeout involved,
+       or a float indicating the timeout to use.
+    :param end: Either ``None`` if there is no timeout involved, or
+       a float giving the absolute end time.
+    :return: An updated value for ``timeleft`` (or None)
+    :raises timeout: If ``timeleft`` was given and elapsed while
+       sending this chunk.
+    """
+    data_sent = 0
+    len_data_memory = len(data_memory)
+    started_timer = 0
+    while data_sent < len_data_memory:
+        chunk = data_memory[data_sent:]
+        if timeleft is None:
+            data_sent += socket.send(chunk, flags)
+        elif started_timer and timeleft <= 0:
+            # Check before sending to guarantee a check
+            # happens even if each chunk successfully sends its data
+            # (especially important for SSL sockets since they have large
+            # buffers). But only do this if we've actually tried to
+            # send something once to avoid spurious timeouts on non-blocking
+            # sockets.
+            raise timeout('timed out')
+        else:
+            started_timer = 1
+            data_sent += socket.send(chunk, flags, timeout=timeleft)
+            timeleft = end - time.time()
+
+    return timeleft
+
+def _sendall(socket, data_memory, flags,
+             SOL_SOCKET=__socket__.SOL_SOCKET,  # pylint:disable=no-member
+             SO_SNDBUF=__socket__.SO_SNDBUF):  # pylint:disable=no-member
+    """
+    Send the *data_memory* (which should be a memoryview)
+    using the gevent *socket*, performing well on PyPy.
+    """
+
+    # On PyPy up through 5.10.0, both PyPy2 and PyPy3, subviews
+    # (slices) of a memoryview() object copy the underlying bytes the
+    # first time the builtin socket.send() method is called. On a
+    # non-blocking socket (that thus calls socket.send() many times)
+    # with a large input, this results in many repeated copies of an
+    # ever smaller string, depending on the networking buffering. For
+    # example, if each send() can process 1MB of a 50MB input, and we
+    # naively pass the entire remaining subview each time, we'd copy
+    # 49MB, 48MB, 47MB, etc, thus completely killing performance. To
+    # workaround this problem, we work in reasonable, fixed-size
+    # chunks. This results in a 10x improvement to bench_sendall.py,
+    # while having no measurable impact on CPython (since it doesn't
+    # copy at all the only extra overhead is a few python function
+    # calls, which is negligible for large inputs).
+
+    # On one macOS machine, PyPy3 5.10.1 produced ~ 67.53 MB/s before this change,
+    # and ~ 616.01 MB/s after.
+
+    # See https://bitbucket.org/pypy/pypy/issues/2091/non-blocking-socketsend-slow-gevent
+
+    # Too small of a chunk (the socket's buf size is usually too
+    # small) results in reduced perf due to *too many* calls to send and too many
+    # small copies. With a buffer of 143K (the default on my system), for
+    # example, bench_sendall.py yields ~264MB/s, while using 1MB yields
+    # ~653MB/s (matching CPython). 1MB is arbitrary and might be better
+    # chosen, say, to match a page size?
+
+    len_data_memory = len(data_memory)
+    if not len_data_memory:
+        # Don't try to send empty data at all, no point, and breaks ssl
+        # See issue 719
+        return 0
+
+
+    chunk_size = max(socket.getsockopt(SOL_SOCKET, SO_SNDBUF), 1024 * 1024)
+
+    data_sent = 0
+    end = None
+    timeleft = None
+    if socket.timeout is not None:
+        timeleft = socket.timeout
+        end = time.time() + timeleft
+
+    while data_sent < len_data_memory:
+        chunk_end = min(data_sent + chunk_size, len_data_memory)
+        chunk = data_memory[data_sent:chunk_end]
+
+        timeleft = __send_chunk(socket, chunk, flags, timeleft, end)
+        data_sent += len(chunk) # Guaranteed it sent the whole thing
+
+# pylint:disable=no-member
+_RESOLVABLE_FAMILIES = (__socket__.AF_INET,)
+if __socket__.has_ipv6:
+    _RESOLVABLE_FAMILIES += (__socket__.AF_INET6,)
+
+def _resolve_addr(sock, address):
+    # Internal method: resolve the AF_INET[6] address using
+    # getaddrinfo.
+    if sock.family not in _RESOLVABLE_FAMILIES or not isinstance(address, tuple):
+        return address
+    # address is (host, port) (ipv4) or (host, port, flowinfo, scopeid) (ipv6).
+
+    # We don't pass the port to getaddrinfo because the C
+    # socket module doesn't either (on some systems its
+    # illegal to do that without also passing socket type and
+    # protocol). Instead we join the port back at the end.
+    # See https://github.com/gevent/gevent/issues/1252
+    host, port = address[:2]
+    r = getaddrinfo(host, None, sock.family)
+    address = r[0][-1]
+    if len(address) == 2:
+        address = (address[0], port)
+    else:
+        address = (address[0], port, address[2], address[3])
+    return address

python/gevent/_ssl2.py

@@ -1,10 +1,15 @@
 # Wrapper module for _ssl. Written by Bill Janssen.
 # Ported to gevent by Denis Bilenko.
-"""SSL wrapper for socket objects on Python 2.7.8 and below.
+"""
+SSL wrapper for socket objects on Python 2.7.8 and below.
 
 For the documentation, refer to :mod:`ssl` module manual.
 
 This module implements cooperative SSL socket wrappers.
+
+.. deprecated:: 1.3
+   This module is not secure. Support for Python versions
+   with only this level of SSL will be dropped in gevent 1.4.
 """
 
 from __future__ import absolute_import
@@ -25,10 +30,12 @@ from gevent._compat import PYPY
 from gevent._util import copy_globals
 
 
-__implements__ = ['SSLSocket',
-                  'wrap_socket',
-                  'get_server_certificate',
-                  'sslwrap_simple']
+__implements__ = [
+    'SSLSocket',
+    'wrap_socket',
+    'get_server_certificate',
+    'sslwrap_simple',
+]
 
 # Import all symbols from Python's ssl.py, except those that we are implementing
 # and "private" symbols.
@@ -102,7 +109,7 @@ class SSLSocket(socket):
             except SSLError as ex:
                 if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:
                     return ''
-                elif ex.args[0] == SSL_ERROR_WANT_READ:
+                if ex.args[0] == SSL_ERROR_WANT_READ:
                     if self.timeout == 0.0:
                         raise
                     sys.exc_clear()
@@ -204,8 +211,7 @@ class SSLSocket(socket):
                     self.__class__)
             # QQQ Shouldn't we wrap the SSL_WANT_READ errors as socket.timeout errors to match socket.recv's behavior?
             return self.read(buflen)
-        else:
-            return socket.recv(self, buflen, flags)
+        return socket.recv(self, buflen, flags)
 
     def recv_into(self, buffer, nbytes=None, flags=0):
         if buffer and (nbytes is None):
@@ -261,7 +267,7 @@ class SSLSocket(socket):
             except SSLError as ex:
                 if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:
                     return ''
-                elif ex.args[0] == SSL_ERROR_WANT_READ:
+                if ex.args[0] == SSL_ERROR_WANT_READ:
                     if self.timeout == 0.0:
                         raise
                     sys.exc_clear()
@@ -275,12 +281,11 @@ class SSLSocket(socket):
                     raise
 
     def unwrap(self):
-        if self._sslobj:
-            s = self._sslobj_shutdown()
-            self._sslobj = None
-            return socket(_sock=s)
-        else:
+        if not self._sslobj:
             raise ValueError("No SSL wrapper around " + str(self))
+        s = self._sslobj_shutdown()
+        self._sslobj = None
+        return socket(_sock=s)
 
     def shutdown(self, how):
         self._sslobj = None
@@ -431,6 +436,6 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None):
 
 def sslwrap_simple(sock, keyfile=None, certfile=None):
     """A replacement for the old socket.ssl function.  Designed
-    for compatability with Python 2.5 and earlier.  Will disappear in
+    for compatibility with Python 2.5 and earlier.  Will disappear in
     Python 3.0."""
     return SSLSocket(sock, keyfile, certfile)

python/gevent/_ssl3.py

@@ -8,11 +8,12 @@ This module implements cooperative SSL socket wrappers.
 """
 # Our import magic sadly makes this warning useless
 # pylint: disable=undefined-variable
+# pylint:disable=no-member
 
 from __future__ import absolute_import
 import ssl as __ssl__
 
-_ssl = __ssl__._ssl # pylint:disable=no-member
+_ssl = __ssl__._ssl
 
 import errno
 from gevent.socket import socket, timeout_default
@@ -44,20 +45,25 @@ orig_SSLContext = __ssl__.SSLContext # pylint:disable=no-member
 
 
 class SSLContext(orig_SSLContext):
+
+    # Added in Python 3.7
+    sslsocket_class = None # SSLSocket is assigned later
+
     def wrap_socket(self, sock, server_side=False,
                     do_handshake_on_connect=True,
                     suppress_ragged_eofs=True,
                     server_hostname=None,
                     session=None):
-        # pylint:disable=arguments-differ
+        # pylint:disable=arguments-differ,not-callable
         # (3.6 adds session)
         # Sadly, using *args and **kwargs doesn't work
-        return SSLSocket(sock=sock, server_side=server_side,
-                         do_handshake_on_connect=do_handshake_on_connect,
-                         suppress_ragged_eofs=suppress_ragged_eofs,
-                         server_hostname=server_hostname,
-                         _context=self,
-                         _session=session)
+        return self.sslsocket_class(
+            sock=sock, server_side=server_side,
+            do_handshake_on_connect=do_handshake_on_connect,
+            suppress_ragged_eofs=suppress_ragged_eofs,
+            server_hostname=server_hostname,
+            _context=self,
+            _session=session)
 
     if not hasattr(orig_SSLContext, 'check_hostname'):
         # Python 3.3 lacks this
@@ -82,6 +88,16 @@ class SSLContext(orig_SSLContext):
         def verify_mode(self, value):
             super(orig_SSLContext, orig_SSLContext).verify_mode.__set__(self, value)
 
+    if hasattr(orig_SSLContext, 'minimum_version'):
+        # Like the above, added in 3.7
+        @orig_SSLContext.minimum_version.setter
+        def minimum_version(self, value):
+            super(orig_SSLContext, orig_SSLContext).minimum_version.__set__(self, value)
+
+        @orig_SSLContext.maximum_version.setter
+        def maximum_version(self, value):
+            super(orig_SSLContext, orig_SSLContext).maximum_version.__set__(self, value)
+
 
 class _contextawaresock(socket._gevent_sock_class): # Python 2: pylint:disable=slots-on-old-class
     # We have to pass the raw stdlib socket to SSLContext.wrap_socket.
@@ -122,6 +138,23 @@ class _contextawaresock(socket._gevent_sock_class): # Python 2: pylint:disable=s
             pass
         raise AttributeError(name)
 
+try:
+    _SSLObject_factory = SSLObject
+except NameError:
+    # 3.4 and below do not have SSLObject, something
+    # we magically import through copy_globals
+    pass
+else:
+    if hasattr(SSLObject, '_create'):
+        # 3.7 is making thing difficult and won't let you
+        # actually construct an object
+        def _SSLObject_factory(sslobj, owner=None, session=None):
+            s = SSLObject.__new__(SSLObject)
+            s._sslobj = sslobj
+            s._sslobj.owner = owner or s
+            if session is not None:
+                s._sslobj.session = session
+            return s
 
 class SSLSocket(socket):
     """
@@ -142,6 +175,7 @@ class SSLSocket(socket):
                  server_hostname=None,
                  _session=None, # 3.6
                  _context=None):
+
         # pylint:disable=too-many-locals,too-many-statements,too-many-branches
         if _context:
             self._context = _context
@@ -218,8 +252,9 @@ class SSLSocket(socket):
             try:
                 self._sslobj = self._context._wrap_socket(self._sock, server_side,
                                                           server_hostname)
-                if _session is not None: # 3.6
-                    self._sslobj = SSLObject(self._sslobj, owner=self, session=self._session)
+                if _session is not None: # 3.6+
+                    self._sslobj = _SSLObject_factory(self._sslobj, owner=self,
+                                                      session=self._session)
                 if do_handshake_on_connect:
                     timeout = self.gettimeout()
                     if timeout == 0.0:
@@ -305,8 +340,7 @@ class SSLSocket(socket):
                     if buffer is None:
                         return b''
                     return 0
-                else:
-                    raise
+                raise
 
     def write(self, data):
         """Write DATA to the underlying SSL channel.  Returns
@@ -455,8 +489,7 @@ class SSLSocket(socket):
                 # Python #23804
                 return b''
             return self.read(buflen)
-        else:
-            return socket.recv(self, buflen, flags)
+        return socket.recv(self, buflen, flags)
 
     def recv_into(self, buffer, nbytes=None, flags=0):
         self._checkClosed()
@@ -468,8 +501,7 @@ class SSLSocket(socket):
             if flags != 0:
                 raise ValueError("non-zero flags not allowed in calls to recv_into() on %s" % self.__class__)
             return self.read(nbytes, buffer)
-        else:
-            return socket.recv_into(self, buffer, nbytes, flags)
+        return socket.recv_into(self, buffer, nbytes, flags)
 
     def recvfrom(self, buflen=1024, flags=0):
         self._checkClosed()
@@ -507,31 +539,41 @@ class SSLSocket(socket):
         socket.shutdown(self, how)
 
     def unwrap(self):
-        if self._sslobj:
-            while True:
-                try:
-                    s = self._sslobj.shutdown()
-                    break
-                except SSLWantReadError:
-                    if self.timeout == 0.0:
-                        return 0
-                    self._wait(self._read_event)
-                except SSLWantWriteError:
-                    if self.timeout == 0.0:
-                        return 0
-                    self._wait(self._write_event)
-
-            self._sslobj = None
-            # The return value of shutting down the SSLObject is the
-            # original wrapped socket, i.e., _contextawaresock. But that
-            # object doesn't have the gevent wrapper around it so it can't
-            # be used. We have to wrap it back up with a gevent wrapper.
-            sock = socket(family=s.family, type=s.type, proto=s.proto, fileno=s.fileno())
-            s.detach()
-            return sock
-        else:
+        if not self._sslobj:
             raise ValueError("No SSL wrapper around " + str(self))
 
+        while True:
+            try:
+                s = self._sslobj.shutdown()
+                break
+            except SSLWantReadError:
+                # Callers of this method expect to get a socket
+                # back, so we can't simply return 0, we have
+                # to let these be raised
+                if self.timeout == 0.0:
+                    raise
+                self._wait(self._read_event)
+            except SSLWantWriteError:
+                if self.timeout == 0.0:
+                    raise
+                self._wait(self._write_event)
+
+        self._sslobj = None
+
+        # The return value of shutting down the SSLObject is the
+        # original wrapped socket passed to _wrap_socket, i.e.,
+        # _contextawaresock. But that object doesn't have the
+        # gevent wrapper around it so it can't be used. We have to
+        # wrap it back up with a gevent wrapper.
+        assert s is self._sock
+        # In the stdlib, SSLSocket subclasses socket.socket and passes itself
+        # to _wrap_socket, so it gets itself back. We can't do that, we have to
+        # pass our subclass of _socket.socket, _contextawaresock.
+        # So ultimately we should return ourself.
+
+        # See test_ftplib.py:TestTLS_FTPClass.test_ccc
+        return self
+
     def _real_close(self):
         self._sslobj = None
         # self._closed = True
@@ -553,7 +595,9 @@ class SSLSocket(socket):
                     raise
                 self._wait(self._write_event, timeout_exc=_SSLErrorHandshakeTimeout)
 
-        if self._context.check_hostname:
+        if sys.version_info[:2] < (3, 7) and self._context.check_hostname:
+            # In Python 3.7, the underlying OpenSSL name matching is used.
+            # The version implemented in Python doesn't understand IDNA encoding.
             if not self.server_hostname:
                 raise ValueError("check_hostname needs server_hostname "
                                  "argument")
@@ -567,8 +611,8 @@ class SSLSocket(socket):
         if self._connected:
             raise ValueError("attempt to connect already-connected SSLSocket!")
         self._sslobj = self._context._wrap_socket(self._sock, False, self.server_hostname)
-        if self._session is not None: # 3.6
-            self._sslobj = SSLObject(self._sslobj, owner=self, session=self._session)
+        if self._session is not None: # 3.6+
+            self._sslobj = _SSLObject_factory(self._sslobj, owner=self, session=self._session)
         try:
             if connect_ex:
                 rc = socket.connect_ex(self, addr)
@@ -600,6 +644,7 @@ class SSLSocket(socket):
         SSL channel, and the address of the remote client."""
 
         newsock, addr = socket.accept(self)
+        newsock._drop_events()
         newsock = self._context.wrap_socket(newsock,
                                             do_handshake_on_connect=self.do_handshake_on_connect,
                                             suppress_ragged_eofs=self.suppress_ragged_eofs,
@@ -611,6 +656,9 @@ class SSLSocket(socket):
         if the requested `cb_type` is not supported.  Return bytes of the data
         or None if the data is not available (e.g. before the handshake).
         """
+        if hasattr(self._sslobj, 'get_channel_binding'):
+            # 3.7+, and sslobj is not None
+            return self._sslobj.get_channel_binding(cb_type)
         if cb_type not in CHANNEL_BINDING_TYPES:
             raise ValueError("Unsupported channel binding type")
         if cb_type != "tls-unique":
@@ -620,6 +668,9 @@ class SSLSocket(socket):
         return self._sslobj.tls_unique_cb()
 
 
+# Python does not support forward declaration of types
+SSLContext.sslsocket_class = SSLSocket
+
 # Python 3.2 onwards raise normal timeout errors, not SSLError.
 # See https://bugs.python.org/issue10272
 _SSLErrorReadTimeout = _socket_timeout('The read operation timed out')

python/gevent/_sslgte279.py

@@ -326,8 +326,7 @@ class SSLSocket(socket):
                     if buffer is not None:
                         return 0
                     return b''
-                else:
-                    raise
+                raise
 
     def write(self, data):
         """Write DATA to the underlying SSL channel.  Returns
@@ -456,8 +455,7 @@ class SSLSocket(socket):
             if buflen == 0:
                 return b''
             return self.read(buflen)
-        else:
-            return socket.recv(self, buflen, flags)
+        return socket.recv(self, buflen, flags)
 
     def recv_into(self, buffer, nbytes=None, flags=0):
         self._checkClosed()
@@ -473,16 +471,14 @@ class SSLSocket(socket):
                     "non-zero flags not allowed in calls to recv_into() on %s" %
                     self.__class__)
             return self.read(nbytes, buffer)
-        else:
-            return socket.recv_into(self, buffer, nbytes, flags)
+        return socket.recv_into(self, buffer, nbytes, flags)
 
     def recvfrom(self, buflen=1024, flags=0):
         self._checkClosed()
         if self._sslobj:
             raise ValueError("recvfrom not allowed on instances of %s" %
                              self.__class__)
-        else:
-            return socket.recvfrom(self, buflen, flags)
+        return socket.recvfrom(self, buflen, flags)
 
     def recvfrom_into(self, buffer, nbytes=None, flags=0):
         self._checkClosed()
@@ -536,7 +532,7 @@ class SSLSocket(socket):
             except SSLError as ex:
                 if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:
                     return ''
-                elif ex.args[0] == SSL_ERROR_WANT_READ:
+                if ex.args[0] == SSL_ERROR_WANT_READ:
                     if self.timeout == 0.0:
                         raise
                     sys.exc_clear()
@@ -550,13 +546,16 @@ class SSLSocket(socket):
                     raise
 
     def unwrap(self):
-        if self._sslobj:
-            s = self._sslobj_shutdown()
-            self._sslobj = None
-            return socket(_sock=s) # match _ssl2; critical to drop/reuse here on PyPy
-        else:
+        if not self._sslobj:
             raise ValueError("No SSL wrapper around " + str(self))
 
+        s = self._sslobj_shutdown()
+        self._sslobj = None
+        # match _ssl2; critical to drop/reuse here on PyPy
+        # XXX: _ssl3 returns an SSLSocket. Is that what the standard lib does on
+        # Python 2? Should we do that?
+        return socket(_sock=s)
+
     def _real_close(self):
         self._sslobj = None
         socket._real_close(self) # pylint: disable=no-member
@@ -622,6 +621,7 @@ class SSLSocket(socket):
         SSL channel, and the address of the remote client."""
 
         newsock, addr = socket.accept(self)
+        newsock._drop_events()
         newsock = self._context.wrap_socket(newsock,
                                             do_handshake_on_connect=self.do_handshake_on_connect,
                                             suppress_ragged_eofs=self.suppress_ragged_eofs,

python/gevent/_tblib.py

@@ -30,6 +30,7 @@ Taken verbatim from Jinja2.
 
 https://github.com/mitsuhiko/jinja2/blob/master/jinja2/debug.py#L267
 """
+# pylint:disable=consider-using-dict-comprehension
 #import platform # XXX: gevent cannot import platform at the top level; interferes with monkey patching
 import sys
 

python/gevent/_threading.py

@@ -6,100 +6,35 @@ or not).
 This module is missing 'Thread' class, but includes 'Queue'.
 """
 from __future__ import absolute_import
-try:
-    from Queue import Full, Empty
-except ImportError:
-    from queue import Full, Empty # pylint:disable=import-error
+
 from collections import deque
-import heapq
-from time import time as _time, sleep as _sleep
 
 from gevent import monkey
-from gevent._compat import PY3
+from gevent._compat import thread_mod_name
 
 
-__all__ = ['Condition',
-           'Event',
-           'Lock',
-           'RLock',
-           'Semaphore',
-           'BoundedSemaphore',
-           'Queue',
-           'local',
-           'stack_size']
+__all__ = [
+    'Lock',
+    'Queue',
+]
 
 
-thread_name = '_thread' if PY3 else 'thread'
-start_new_thread, Lock, get_ident, local, stack_size = monkey.get_original(thread_name, [
-    'start_new_thread', 'allocate_lock', 'get_ident', '_local', 'stack_size'])
+start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
+    'start_new_thread', 'allocate_lock', 'get_ident',
+])
 
 
-class RLock(object):
-
-    def __init__(self):
-        self.__block = Lock()
-        self.__owner = None
-        self.__count = 0
-
-    def __repr__(self):
-        owner = self.__owner
-        return "<%s owner=%r count=%d>" % (
-            self.__class__.__name__, owner, self.__count)
-
-    def acquire(self, blocking=1):
-        me = get_ident()
-        if self.__owner == me:
-            self.__count = self.__count + 1
-            return 1
-        rc = self.__block.acquire(blocking)
-        if rc:
-            self.__owner = me
-            self.__count = 1
-        return rc
-
-    __enter__ = acquire
-
-    def release(self):
-        if self.__owner != get_ident():
-            raise RuntimeError("cannot release un-acquired lock")
-        self.__count = count = self.__count - 1
-        if not count:
-            self.__owner = None
-            self.__block.release()
-
-    def __exit__(self, t, v, tb):
-        self.release()
-
-    # Internal methods used by condition variables
-
-    def _acquire_restore(self, count_owner):
-        count, owner = count_owner
-        self.__block.acquire()
-        self.__count = count
-        self.__owner = owner
-
-    def _release_save(self):
-        count = self.__count
-        self.__count = 0
-        owner = self.__owner
-        self.__owner = None
-        self.__block.release()
-        return (count, owner)
-
-    def _is_owned(self):
-        return self.__owner == get_ident()
+# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
+# pylint:disable=assignment-from-no-return
 
 
-class Condition(object):
+class _Condition(object):
     # pylint:disable=method-hidden
 
-    def __init__(self, lock=None):
-        if lock is None:
-            lock = RLock()
+    def __init__(self, lock):
         self.__lock = lock
-        # Export the lock's acquire() and release() methods
-        self.acquire = lock.acquire
-        self.release = lock.release
+        self.__waiters = []
+
         # If the lock defines _release_save() and/or _acquire_restore(),
         # these override the default implementations (which just call
         # release() and acquire() on the lock).  Ditto for _is_owned().
@@ -115,13 +50,12 @@ class Condition(object):
             self._is_owned = lock._is_owned
         except AttributeError:
             pass
-        self.__waiters = []
 
     def __enter__(self):
         return self.__lock.__enter__()
 
-    def __exit__(self, *args):
-        return self.__lock.__exit__(*args)
+    def __exit__(self, t, v, tb):
+        return self.__lock.__exit__(t, v, tb)
 
     def __repr__(self):
         return "<Condition(%s, %d)>" % (self.__lock, len(self.__waiters))
@@ -140,168 +74,47 @@ class Condition(object):
             return False
         return True
 
-    def wait(self, timeout=None):
-        if not self._is_owned():
-            raise RuntimeError("cannot wait on un-acquired lock")
+    def wait(self):
+        # The condition MUST be owned, but we don't check that.
         waiter = Lock()
         waiter.acquire()
         self.__waiters.append(waiter)
         saved_state = self._release_save()
         try:    # restore state no matter what (e.g., KeyboardInterrupt)
-            if timeout is None:
-                waiter.acquire()
-            else:
-                # Balancing act:  We can't afford a pure busy loop, so we
-                # have to sleep; but if we sleep the whole timeout time,
-                # we'll be unresponsive.  The scheme here sleeps very
-                # little at first, longer as time goes on, but never longer
-                # than 20 times per second (or the timeout time remaining).
-                endtime = _time() + timeout
-                delay = 0.0005  # 500 us -> initial delay of 1 ms
-                while True:
-                    gotit = waiter.acquire(0)
-                    if gotit:
-                        break
-                    remaining = endtime - _time()
-                    if remaining <= 0:
-                        break
-                    delay = min(delay * 2, remaining, .05)
-                    _sleep(delay)
-                if not gotit:
-                    try:
-                        self.__waiters.remove(waiter)
-                    except ValueError:
-                        pass
+            waiter.acquire() # Block on the native lock
         finally:
             self._acquire_restore(saved_state)
 
-    def notify(self, n=1):
-        if not self._is_owned():
-            raise RuntimeError("cannot notify on un-acquired lock")
-        __waiters = self.__waiters
-        waiters = __waiters[:n]
-        if not waiters:
-            return
-        for waiter in waiters:
-            waiter.release()
-            try:
-                __waiters.remove(waiter)
-            except ValueError:
-                pass
-
-    def notify_all(self):
-        self.notify(len(self.__waiters))
-
-
-class Semaphore(object):
-
-    # After Tim Peters' semaphore class, but not quite the same (no maximum)
-
-    def __init__(self, value=1):
-        if value < 0:
-            raise ValueError("semaphore initial value must be >= 0")
-        self.__cond = Condition(Lock())
-        self.__value = value
-
-    def acquire(self, blocking=1):
-        rc = False
-        self.__cond.acquire()
-        while self.__value == 0:
-            if not blocking:
-                break
-            self.__cond.wait()
+    def notify_one(self):
+        # The condition MUST be owned, but we don't check that.
+        try:
+            waiter = self.__waiters.pop()
+        except IndexError:
+            # Nobody around
+            pass
         else:
-            self.__value = self.__value - 1
-            rc = True
-        self.__cond.release()
-        return rc
-
-    __enter__ = acquire
-
-    def release(self):
-        self.__cond.acquire()
-        self.__value = self.__value + 1
-        self.__cond.notify()
-        self.__cond.release()
-
-    def __exit__(self, t, v, tb):
-        self.release()
+            waiter.release()
 
 
-class BoundedSemaphore(Semaphore):
-    """Semaphore that checks that # releases is <= # acquires"""
-    def __init__(self, value=1):
-        Semaphore.__init__(self, value)
-        self._initial_value = value
+class Queue(object):
+    """Create a queue object.
 
-    def release(self):
-        if self.Semaphore__value >= self._initial_value: # pylint:disable=no-member
-            raise ValueError("Semaphore released too many times")
-        return Semaphore.release(self)
+    The queue is always infinite size.
+    """
 
-
-class Event(object):
-
-    # After Tim Peters' event class (without is_posted())
+    __slots__ = ('_queue', '_mutex', '_not_empty', 'unfinished_tasks')
 
     def __init__(self):
-        self.__cond = Condition(Lock())
-        self.__flag = False
-
-    def _reset_internal_locks(self):
-        # private!  called by Thread._reset_internal_locks by _after_fork()
-        self.__cond.__init__()
-
-    def is_set(self):
-        return self.__flag
-
-    def set(self):
-        self.__cond.acquire()
-        try:
-            self.__flag = True
-            self.__cond.notify_all()
-        finally:
-            self.__cond.release()
-
-    def clear(self):
-        self.__cond.acquire()
-        try:
-            self.__flag = False
-        finally:
-            self.__cond.release()
-
-    def wait(self, timeout=None):
-        self.__cond.acquire()
-        try:
-            if not self.__flag:
-                self.__cond.wait(timeout)
-            return self.__flag
-        finally:
-            self.__cond.release()
-
-
-class Queue: # pylint:disable=old-style-class
-    """Create a queue object with a given maximum size.
-
-    If maxsize is <= 0, the queue size is infinite.
-    """
-    def __init__(self, maxsize=0):
-        self.maxsize = maxsize
-        self._init(maxsize)
+        self._queue = deque()
         # mutex must be held whenever the queue is mutating.  All methods
         # that acquire mutex must release it before returning.  mutex
         # is shared between the three conditions, so acquiring and
         # releasing the conditions also acquires and releases mutex.
-        self.mutex = Lock()
+        self._mutex = Lock()
         # Notify not_empty whenever an item is added to the queue; a
         # thread waiting to get is notified then.
-        self.not_empty = Condition(self.mutex)
-        # Notify not_full whenever an item is removed from the queue;
-        # a thread waiting to put is notified then.
-        self.not_full = Condition(self.mutex)
-        # Notify all_tasks_done whenever the number of unfinished tasks
-        # drops to zero; thread waiting to join() is notified to resume
-        self.all_tasks_done = Condition(self.mutex)
+        self._not_empty = _Condition(self._mutex)
+
         self.unfinished_tasks = 0
 
     def task_done(self):
@@ -318,198 +131,38 @@ class Queue: # pylint:disable=old-style-class
         Raises a ValueError if called more times than there were items
         placed in the queue.
         """
-        self.all_tasks_done.acquire()
-        try:
+        with self._mutex:
             unfinished = self.unfinished_tasks - 1
             if unfinished <= 0:
                 if unfinished < 0:
                     raise ValueError('task_done() called too many times')
-                self.all_tasks_done.notify_all()
             self.unfinished_tasks = unfinished
-        finally:
-            self.all_tasks_done.release()
 
-    def join(self):
-        """Blocks until all items in the Queue have been gotten and processed.
-
-        The count of unfinished tasks goes up whenever an item is added to the
-        queue. The count goes down whenever a consumer thread calls task_done()
-        to indicate the item was retrieved and all work on it is complete.
-
-        When the count of unfinished tasks drops to zero, join() unblocks.
-        """
-        self.all_tasks_done.acquire()
-        try:
-            while self.unfinished_tasks:
-                self.all_tasks_done.wait()
-        finally:
-            self.all_tasks_done.release()
-
-    def qsize(self):
+    def qsize(self, len=len):
         """Return the approximate size of the queue (not reliable!)."""
-        self.mutex.acquire()
-        try:
-            return self._qsize()
-        finally:
-            self.mutex.release()
+        return len(self._queue)
 
     def empty(self):
         """Return True if the queue is empty, False otherwise (not reliable!)."""
-        self.mutex.acquire()
-        try:
-            return not self._qsize()
-        finally:
-            self.mutex.release()
+        return not self.qsize()
 
     def full(self):
         """Return True if the queue is full, False otherwise (not reliable!)."""
-        self.mutex.acquire()
-        try:
-            if self.maxsize <= 0:
-                return False
-            if self.maxsize >= self._qsize():
-                return True
-        finally:
-            self.mutex.release()
+        return False
 
-    def put(self, item, block=True, timeout=None):
+    def put(self, item):
         """Put an item into the queue.
-
-        If optional args 'block' is true and 'timeout' is None (the default),
-        block if necessary until a free slot is available. If 'timeout' is
-        a positive number, it blocks at most 'timeout' seconds and raises
-        the Full exception if no free slot was available within that time.
-        Otherwise ('block' is false), put an item on the queue if a free slot
-        is immediately available, else raise the Full exception ('timeout'
-        is ignored in that case).
         """
-        self.not_full.acquire()
-        try:
-            if self.maxsize > 0:
-                if not block:
-                    if self._qsize() >= self.maxsize:
-                        raise Full
-                elif timeout is None:
-                    while self._qsize() >= self.maxsize:
-                        self.not_full.wait()
-                elif timeout < 0:
-                    raise ValueError("'timeout' must be a positive number")
-                else:
-                    endtime = _time() + timeout
-                    while self._qsize() >= self.maxsize:
-                        remaining = endtime - _time()
-                        if remaining <= 0.0:
-                            raise Full
-                        self.not_full.wait(remaining)
-            self._put(item)
+        with self._not_empty:
+            self._queue.append(item)
             self.unfinished_tasks += 1
-            self.not_empty.notify()
-        finally:
-            self.not_full.release()
+            self._not_empty.notify_one()
 
-    def put_nowait(self, item):
-        """Put an item into the queue without blocking.
-
-        Only enqueue the item if a free slot is immediately available.
-        Otherwise raise the Full exception.
-        """
-        return self.put(item, False)
-
-    def get(self, block=True, timeout=None):
+    def get(self):
         """Remove and return an item from the queue.
-
-        If optional args 'block' is true and 'timeout' is None (the default),
-        block if necessary until an item is available. If 'timeout' is
-        a positive number, it blocks at most 'timeout' seconds and raises
-        the Empty exception if no item was available within that time.
-        Otherwise ('block' is false), return an item if one is immediately
-        available, else raise the Empty exception ('timeout' is ignored
-        in that case).
         """
-        self.not_empty.acquire()
-        try:
-            if not block:
-                if not self._qsize():
-                    raise Empty
-            elif timeout is None:
-                while not self._qsize():
-                    self.not_empty.wait()
-            elif timeout < 0:
-                raise ValueError("'timeout' must be a positive number")
-            else:
-                endtime = _time() + timeout
-                while not self._qsize():
-                    remaining = endtime - _time()
-                    if remaining <= 0.0:
-                        raise Empty
-                    self.not_empty.wait(remaining)
-            item = self._get()
-            self.not_full.notify()
+        with self._not_empty:
+            while not self._queue:
+                self._not_empty.wait()
+            item = self._queue.popleft()
             return item
-        finally:
-            self.not_empty.release()
-
-    def get_nowait(self):
-        """Remove and return an item from the queue without blocking.
-
-        Only get an item if one is immediately available. Otherwise
-        raise the Empty exception.
-        """
-        return self.get(False)
-
-    # Override these methods to implement other queue organizations
-    # (e.g. stack or priority queue).
-    # These will only be called with appropriate locks held
-
-    # Initialize the queue representation
-    def _init(self, maxsize):
-        # pylint:disable=unused-argument
-        self.queue = deque()
-
-    def _qsize(self, len=len):
-        return len(self.queue)
-
-    # Put a new item in the queue
-    def _put(self, item):
-        self.queue.append(item)
-
-    # Get an item from the queue
-    def _get(self):
-        return self.queue.popleft()
-
-
-class PriorityQueue(Queue):
-    '''Variant of Queue that retrieves open entries in priority order (lowest first).
-
-    Entries are typically tuples of the form:  (priority number, data).
-    '''
-
-    def _init(self, maxsize):
-        self.queue = []
-
-    def _qsize(self, len=len):
-        return len(self.queue)
-
-    def _put(self, item, heappush=heapq.heappush):
-        # pylint:disable=arguments-differ
-        heappush(self.queue, item)
-
-    def _get(self, heappop=heapq.heappop):
-        # pylint:disable=arguments-differ
-        return heappop(self.queue)
-
-
-class LifoQueue(Queue):
-    '''Variant of Queue that retrieves most recently added entries first.'''
-
-    def _init(self, maxsize):
-        self.queue = []
-
-    def _qsize(self, len=len):
-        return len(self.queue)
-
-    def _put(self, item):
-        self.queue.append(item)
-
-    def _get(self):
-        return self.queue.pop()

python/gevent/_tracer.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2018 gevent. See LICENSE for details.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+from __future__ import print_function, absolute_import, division
+
+import sys
+import traceback
+
+from greenlet import settrace
+from greenlet import getcurrent
+
+from gevent.util import format_run_info
+
+from gevent._compat import perf_counter
+from gevent._util import gmctime
+
+
+__all__ = [
+    'GreenletTracer',
+    'HubSwitchTracer',
+    'MaxSwitchTracer',
+]
+
+# Recall these classes are cython compiled, so
+# class variable declarations are bad.
+
+
+class GreenletTracer(object):
+    def __init__(self):
+        # A counter, incremented by the greenlet trace function
+        # we install on every greenlet switch. This is reset when the
+        # periodic monitoring thread runs.
+
+        self.greenlet_switch_counter = 0
+
+        # The greenlet last switched to.
+        self.active_greenlet = None
+
+        # The trace function that was previously installed,
+        # if any.
+        # NOTE: Calling a class instance is cheaper than
+        # calling a bound method (at least when compiled with cython)
+        # even when it redirects to another function.
+        prev_trace = settrace(self)
+
+        self.previous_trace_function = prev_trace
+
+        self._killed = False
+
+    def kill(self):
+        # Must be called in the monitored thread.
+        if not self._killed:
+            self._killed = True
+            settrace(self.previous_trace_function)
+            self.previous_trace_function = None
+
+    def _trace(self, event, args):
+        # This function runs in the thread we are monitoring.
+        self.greenlet_switch_counter += 1
+        if event in ('switch', 'throw'):
+            # args is (origin, target). This is the only defined
+            # case
+            self.active_greenlet = args[1]
+        else:
+            self.active_greenlet = None
+        if self.previous_trace_function is not None:
+            self.previous_trace_function(event, args)
+
+    def __call__(self, event, args):
+        return self._trace(event, args)
+
+    def did_block_hub(self, hub):
+        # Check to see if we have blocked since the last call to this
+        # method. Returns a true value if we blocked (not in the hub),
+        # a false value if everything is fine.
+
+        # This may be called in the same thread being traced or a
+        # different thread; if a different thread, there is a race
+        # condition with this being incremented in the thread we're
+        # monitoring, but probably not often enough to lead to
+        # annoying false positives.
+
+        active_greenlet = self.active_greenlet
+        did_switch = self.greenlet_switch_counter != 0
+        self.greenlet_switch_counter = 0
+
+        if did_switch or active_greenlet is None or active_greenlet is hub:
+            # Either we switched, or nothing is running (we got a
+            # trace event we don't know about or were requested to
+            # ignore), or we spent the whole time in the hub, blocked
+            # for IO. Nothing to report.
+            return False
+        return True, active_greenlet
+
+    def ignore_current_greenlet_blocking(self):
+        # Don't pay attention to the current greenlet.
+        self.active_greenlet = None
+
+    def monitor_current_greenlet_blocking(self):
+        self.active_greenlet = getcurrent()
+
+    def did_block_hub_report(self, hub, active_greenlet, format_kwargs):
+        report = ['=' * 80,
+                  '\n%s : Greenlet %s appears to be blocked' %
+                  (gmctime(), active_greenlet)]
+        report.append("    Reported by %s" % (self,))
+        try:
+            frame = sys._current_frames()[hub.thread_ident]
+        except KeyError:
+            # The thread holding the hub has died. Perhaps we shouldn't
+            # even report this?
+            stack = ["Unknown: No thread found for hub %r\n" % (hub,)]
+        else:
+            stack = traceback.format_stack(frame)
+        report.append('Blocked Stack (for thread id %s):' % (hex(hub.thread_ident),))
+        report.append(''.join(stack))
+        report.append("Info:")
+        report.extend(format_run_info(**format_kwargs))
+
+        return report
+
+
+class _HubTracer(GreenletTracer):
+    def __init__(self, hub, max_blocking_time):
+        GreenletTracer.__init__(self)
+        self.max_blocking_time = max_blocking_time
+        self.hub = hub
+
+    def kill(self):
+        self.hub = None
+        GreenletTracer.kill(self)
+
+
+class HubSwitchTracer(_HubTracer):
+    # A greenlet tracer that records the last time we switched *into* the hub.
+
+    def __init__(self, hub, max_blocking_time):
+        _HubTracer.__init__(self, hub, max_blocking_time)
+        self.last_entered_hub = 0
+
+    def _trace(self, event, args):
+        GreenletTracer._trace(self, event, args)
+        if self.active_greenlet is self.hub:
+            self.last_entered_hub = perf_counter()
+
+    def did_block_hub(self, hub):
+        if perf_counter() - self.last_entered_hub > self.max_blocking_time:
+            return True, self.active_greenlet
+
+
+class MaxSwitchTracer(_HubTracer):
+    # A greenlet tracer that records the maximum time between switches,
+    # not including time spent in the hub.
+
+    def __init__(self, hub, max_blocking_time):
+        _HubTracer.__init__(self, hub, max_blocking_time)
+        self.last_switch = perf_counter()
+        self.max_blocking = 0
+
+    def _trace(self, event, args):
+        old_active = self.active_greenlet
+        GreenletTracer._trace(self, event, args)
+        if old_active is not self.hub and old_active is not None:
+            # If we're switching out of the hub, the blocking
+            # time doesn't count.
+            switched_at = perf_counter()
+            self.max_blocking = max(self.max_blocking,
+                                    switched_at - self.last_switch)
+
+    def did_block_hub(self, hub):
+        if self.max_blocking == 0:
+            # We never switched. Check the time now
+            self.max_blocking = perf_counter() - self.last_switch
+
+        if self.max_blocking > self.max_blocking_time:
+            return True, self.active_greenlet
+
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__tracer')

python/gevent/_util.py

@@ -5,6 +5,8 @@ internal gevent utilities, not for external use.
 
 from __future__ import print_function, absolute_import, division
 
+from functools import update_wrapper
+
 from gevent._compat import iteritems
 
 
@@ -29,21 +31,22 @@ def copy_globals(source,
                  dunder_names_to_keep=('__implements__', '__all__', '__imports__'),
                  cleanup_globs=True):
     """
-    Copy attributes defined in `source.__dict__` to the dictionary in globs
-    (which should be the caller's globals()).
+    Copy attributes defined in ``source.__dict__`` to the dictionary
+    in globs (which should be the caller's :func:`globals`).
 
-    Names that start with `__` are ignored (unless they are in
+    Names that start with ``__`` are ignored (unless they are in
     *dunder_names_to_keep*). Anything found in *names_to_ignore* is
     also ignored.
 
-    If *only_names* is given, only those attributes will be considered.
-    In this case, *ignore_missing_names* says whether or not to raise an AttributeError
-    if one of those names can't be found.
+    If *only_names* is given, only those attributes will be
+    considered. In this case, *ignore_missing_names* says whether or
+    not to raise an :exc:`AttributeError` if one of those names can't
+    be found.
 
-    If cleanup_globs has a true value, then common things imported but not used
-    at runtime are removed, including this function.
+    If *cleanup_globs* has a true value, then common things imported but
+    not used at runtime are removed, including this function.
 
-    Returns a list of the names copied
+    Returns a list of the names copied; this should be assigned to ``__imports__``.
     """
     if only_names:
         if ignore_missing_names:
@@ -70,6 +73,47 @@ def copy_globals(source,
 
     return copied
 
+def import_c_accel(globs, cname):
+    """
+    Import the C-accelerator for the __name__
+    and copy its globals.
+    """
+
+    name = globs.get('__name__')
+
+    if not name or name == cname:
+        # Do nothing if we're being exec'd as a file (no name)
+        # or we're running from the C extension
+        return
+
+
+    from gevent._compat import PURE_PYTHON
+    if PURE_PYTHON:
+        return
+
+    import importlib
+    import warnings
+    with warnings.catch_warnings():
+        # Python 3.7 likes to produce
+        # "ImportWarning: can't resolve
+        #   package from __spec__ or __package__, falling back on
+        #   __name__ and __path__"
+        # when we load cython compiled files. This is probably a bug in
+        # Cython, but it doesn't seem to have any consequences, it's
+        # just annoying to see and can mess up our unittests.
+        warnings.simplefilter('ignore', ImportWarning)
+        mod = importlib.import_module(cname)
+
+    # By adopting the entire __dict__, we get a more accurate
+    # __file__ and module repr, plus we don't leak any imported
+    # things we no longer need.
+    globs.clear()
+    globs.update(mod.__dict__)
+
+    if 'import_c_accel' in globs:
+        del globs['import_c_accel']
+
+
 class Lazy(object):
     """
     A non-data descriptor used just like @property. The
@@ -79,6 +123,7 @@ class Lazy(object):
     """
     def __init__(self, func):
         self.data = (func, func.__name__)
+        update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:
@@ -98,9 +143,36 @@ class readproperty(object):
 
     def __init__(self, func):
         self.func = func
+        update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:
             return self
 
         return self.func(inst)
+
+def gmctime():
+    """
+    Returns the current time as a string in RFC3339 format.
+    """
+    import time
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+try:
+    from zope.interface import Interface
+    from zope.interface import implementer
+    from zope.interface import Attribute
+except ImportError:
+    class Interface(object):
+        pass
+    def implementer(_iface):
+        def dec(c):
+            return c
+        return dec
+
+    def Attribute(s):
+        return s
+
+Interface = Interface
+implementer = implementer
+Attribute = Attribute

python/gevent/_util_py2.py

@@ -1,7 +1,23 @@
-# this produces syntax error on Python3
+import sys
 
 __all__ = ['reraise']
 
 
-def reraise(type, value, tb):
-    raise type, value, tb
+def exec_(_code_, _globs_=None, _locs_=None):
+    """Execute code in a namespace."""
+    if _globs_ is None:
+        frame = sys._getframe(1)
+        _globs_ = frame.f_globals
+        if _locs_ is None:
+            _locs_ = frame.f_locals
+        del frame
+    elif _locs_ is None:
+        _locs_ = _globs_
+    exec("""exec _code_ in _globs_, _locs_""")
+
+exec_("""def reraise(tp, value, tb=None):
+    try:
+        raise tp, value, tb
+    finally:
+        tb = None
+""")

python/gevent/_waiter.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+# copyright 2018 gevent
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+"""
+Low-level waiting primitives.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+from gevent._hub_local import get_hub_noargs as get_hub
+from gevent.exceptions import ConcurrentObjectUseError
+
+__all__ = [
+    'Waiter',
+]
+
+_NONE = object()
+
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+
+
+class Waiter(object):
+    """
+    A low level communication utility for greenlets.
+
+    Waiter is a wrapper around greenlet's ``switch()`` and ``throw()`` calls that makes them somewhat safer:
+
+    * switching will occur only if the waiting greenlet is executing :meth:`get` method currently;
+    * any error raised in the greenlet is handled inside :meth:`switch` and :meth:`throw`
+    * if :meth:`switch`/:meth:`throw` is called before the receiver calls :meth:`get`, then :class:`Waiter`
+      will store the value/exception. The following :meth:`get` will return the value/raise the exception.
+
+    The :meth:`switch` and :meth:`throw` methods must only be called from the :class:`Hub` greenlet.
+    The :meth:`get` method must be called from a greenlet other than :class:`Hub`.
+
+        >>> result = Waiter()
+        >>> timer = get_hub().loop.timer(0.1)
+        >>> timer.start(result.switch, 'hello from Waiter')
+        >>> result.get() # blocks for 0.1 seconds
+        'hello from Waiter'
+        >>> timer.close()
+
+    If switch is called before the greenlet gets a chance to call :meth:`get` then
+    :class:`Waiter` stores the value.
+
+        >>> result = Waiter()
+        >>> timer = get_hub().loop.timer(0.1)
+        >>> timer.start(result.switch, 'hi from Waiter')
+        >>> sleep(0.2)
+        >>> result.get() # returns immediately without blocking
+        'hi from Waiter'
+        >>> timer.close()
+
+    .. warning::
+
+        This a limited and dangerous way to communicate between
+        greenlets. It can easily leave a greenlet unscheduled forever
+        if used incorrectly. Consider using safer classes such as
+        :class:`gevent.event.Event`, :class:`gevent.event.AsyncResult`,
+        or :class:`gevent.queue.Queue`.
+    """
+
+    __slots__ = ['hub', 'greenlet', 'value', '_exception']
+
+    def __init__(self, hub=None):
+        self.hub = get_hub() if hub is None else hub
+        self.greenlet = None
+        self.value = None
+        self._exception = _NONE
+
+    def clear(self):
+        self.greenlet = None
+        self.value = None
+        self._exception = _NONE
+
+    def __str__(self):
+        if self._exception is _NONE:
+            return '<%s greenlet=%s>' % (type(self).__name__, self.greenlet)
+        if self._exception is None:
+            return '<%s greenlet=%s value=%r>' % (type(self).__name__, self.greenlet, self.value)
+        return '<%s greenlet=%s exc_info=%r>' % (type(self).__name__, self.greenlet, self.exc_info)
+
+    def ready(self):
+        """Return true if and only if it holds a value or an exception"""
+        return self._exception is not _NONE
+
+    def successful(self):
+        """Return true if and only if it is ready and holds a value"""
+        return self._exception is None
+
+    @property
+    def exc_info(self):
+        "Holds the exception info passed to :meth:`throw` if :meth:`throw` was called. Otherwise ``None``."
+        if self._exception is not _NONE:
+            return self._exception
+
+    def switch(self, value):
+        """
+        Switch to the greenlet if one's available. Otherwise store the
+        *value*.
+
+        .. versionchanged:: 1.3b1
+           The *value* is no longer optional.
+        """
+        greenlet = self.greenlet
+        if greenlet is None:
+            self.value = value
+            self._exception = None
+        else:
+            if getcurrent() is not self.hub: # pylint:disable=undefined-variable
+                raise AssertionError("Can only use Waiter.switch method from the Hub greenlet")
+            switch = greenlet.switch
+            try:
+                switch(value)
+            except: # pylint:disable=bare-except
+                self.hub.handle_error(switch, *sys.exc_info())
+
+    def switch_args(self, *args):
+        return self.switch(args)
+
+    def throw(self, *throw_args):
+        """Switch to the greenlet with the exception. If there's no greenlet, store the exception."""
+        greenlet = self.greenlet
+        if greenlet is None:
+            self._exception = throw_args
+        else:
+            if getcurrent() is not self.hub: # pylint:disable=undefined-variable
+                raise AssertionError("Can only use Waiter.switch method from the Hub greenlet")
+            throw = greenlet.throw
+            try:
+                throw(*throw_args)
+            except: # pylint:disable=bare-except
+                self.hub.handle_error(throw, *sys.exc_info())
+
+    def get(self):
+        """If a value/an exception is stored, return/raise it. Otherwise until switch() or throw() is called."""
+        if self._exception is not _NONE:
+            if self._exception is None:
+                return self.value
+            getcurrent().throw(*self._exception) # pylint:disable=undefined-variable
+        else:
+            if self.greenlet is not None:
+                raise ConcurrentObjectUseError('This Waiter is already used by %r' % (self.greenlet, ))
+            self.greenlet = getcurrent() # pylint:disable=undefined-variable
+            try:
+                return self.hub.switch()
+            finally:
+                self.greenlet = None
+
+    def __call__(self, source):
+        if source.exception is None:
+            self.switch(source.value)
+        else:
+            self.throw(source.exception)
+
+    # can also have a debugging version, that wraps the value in a tuple (self, value) in switch()
+    # and unwraps it in wait() thus checking that switch() was indeed called
+
+
+
+class MultipleWaiter(Waiter):
+    """
+    An internal extension of Waiter that can be used if multiple objects
+    must be waited on, and there is a chance that in between waits greenlets
+    might be switched out. All greenlets that switch to this waiter
+    will have their value returned.
+
+    This does not handle exceptions or throw methods.
+    """
+    __slots__ = ['_values']
+
+    def __init__(self, hub=None):
+        Waiter.__init__(self, hub)
+        # we typically expect a relatively small number of these to be outstanding.
+        # since we pop from the left, a deque might be slightly
+        # more efficient, but since we're in the hub we avoid imports if
+        # we can help it to better support monkey-patching, and delaying the import
+        # here can be impractical (see https://github.com/gevent/gevent/issues/652)
+        self._values = list()
+
+    def switch(self, value):
+        self._values.append(value)
+        Waiter.switch(self, True)
+
+    def get(self):
+        if not self._values:
+            Waiter.get(self)
+            Waiter.clear(self)
+
+        return self._values.pop(0)
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__waiter')

python/gevent/ares.py

@@ -0,0 +1,10 @@
+"""Backwards compatibility alias for :mod:`gevent.resolver.cares`.
+
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.cares`
+"""
+
+from gevent.resolver.cares import * # pylint:disable=wildcard-import,unused-wildcard-import,
+import gevent.resolver.cares as _cares
+__all__ = _cares.__all__ # pylint:disable=c-extension-no-member
+del _cares

python/gevent/backdoor.py

@@ -178,6 +178,10 @@ class _fileobject(object):
     def __getattr__(self, name):
         return getattr(self._fobj, name)
 
+    def close(self):
+        self._fobj.close()
+        self._sock.close()
+
     def write(self, data):
         if not isinstance(data, bytes):
             data = data.encode('utf-8')

python/gevent/baseserver.py

@@ -163,9 +163,11 @@ class BaseServer(object):
     def stop_accepting(self):
         if self._watcher is not None:
             self._watcher.stop()
+            self._watcher.close()
             self._watcher = None
         if self._timer is not None:
             self._timer.stop()
+            self._timer.close()
             self._timer = None
 
     def do_handle(self, *args):
@@ -398,5 +400,5 @@ def _parse_address(address):
 def parse_address(address):
     try:
         return _parse_address(address)
-    except ValueError as ex:
+    except ValueError as ex: # pylint:disable=try-except-raise
         raise ValueError('Failed to parse address %r: %s' % (address, ex))

python/gevent/builtins.py

@@ -2,10 +2,13 @@
 """gevent friendly implementations of builtin functions."""
 from __future__ import absolute_import
 
-import imp # deprecated since 3.4; issues PendingDeprecationWarning in 3.5
 import sys
 import weakref
+
 from gevent.lock import RLock
+from gevent._compat import imp_acquire_lock
+from gevent._compat import imp_release_lock
+
 
 # Normally we'd have the "expected" case inside the try
 # (Python 3, because Python 3 is the way forward). But
@@ -14,15 +17,15 @@ from gevent.lock import RLock
 # So we test for the old, deprecated version first
 
 try: # Py2
-    import __builtin__ as builtins
+    import __builtin__ as __gbuiltins__
     _allowed_module_name_types = (basestring,) # pylint:disable=undefined-variable
     __target__ = '__builtin__'
 except ImportError:
-    import builtins # pylint: disable=import-error
+    import builtins as __gbuiltins__ # pylint: disable=import-error
     _allowed_module_name_types = (str,)
     __target__ = 'builtins'
 
-_import = builtins.__import__
+_import = __gbuiltins__.__import__
 
 # We need to protect imports both across threads and across greenlets.
 # And the order matters. Note that under 3.4, the global import lock
@@ -86,7 +89,7 @@ def __import__(*args, **kwargs):
         return _import(*args, **kwargs)
 
     module_lock = __module_lock(args[0]) # Get a lock for the module name
-    imp.acquire_lock()
+    imp_acquire_lock()
     try:
         module_lock.acquire()
         try:
@@ -94,7 +97,7 @@ def __import__(*args, **kwargs):
         finally:
             module_lock.release()
     finally:
-        imp.release_lock()
+        imp_release_lock()
     return result
 
 
@@ -120,6 +123,13 @@ def _lock_imports():
 
 if sys.version_info[:2] >= (3, 3):
     __implements__ = []
+    __import__ = _import
 else:
     __implements__ = ['__import__']
 __all__ = __implements__
+
+
+from gevent._util import copy_globals
+
+__imports__ = copy_globals(__gbuiltins__, globals(),
+                           names_to_ignore=__implements__)

python/gevent/core.py

@@ -1,21 +1,19 @@
 # Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
+"""
+Deprecated; this does not reflect all the possible options
+and its interface varies.
+
+.. versionchanged:: 1.3a2
+    Deprecated.
+"""
 from __future__ import absolute_import
 
-import os
+import sys
 
+from gevent._config import config
 from gevent._util import copy_globals
 
-try:
-    if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
-        raise ImportError("Not attempting corecext")
-
-    from gevent.libev import corecext as _core
-except ImportError:
-    if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
-        raise
-
-    # CFFI/PyPy
-    from gevent.libev import corecffi as _core
+_core = sys.modules[config.loop.__module__]
 
 copy_globals(_core, globals())
 

python/gevent/event.py

@@ -1,21 +1,34 @@
 # Copyright (c) 2009-2016 Denis Bilenko, gevent contributors. See LICENSE for details.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,infer_types=True
+
 """Basic synchronization primitives: Event and AsyncResult"""
 from __future__ import print_function
 import sys
-from gevent.hub import get_hub, getcurrent, _NONE
+
+from gevent._util import _NONE
 from gevent._compat import reraise
-from gevent.hub import InvalidSwitchError
-from gevent.timeout import Timeout
 from gevent._tblib import dump_traceback, load_traceback
 
-__all__ = ['Event', 'AsyncResult']
+from gevent._hub_local import get_hub_noargs as get_hub
+
+from gevent.exceptions import InvalidSwitchError
+from gevent.timeout import Timeout
+
+
+__all__ = [
+    'Event',
+    'AsyncResult',
+]
+
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
 
 
 class _AbstractLinkable(object):
     # Encapsulates the standard parts of the linking and notifying protocol
     # common to both repeatable events and one-time events (AsyncResult).
 
-    _notifier = None
+    __slots__ = ('_links', 'hub', '_notifier')
 
     def __init__(self):
         # Also previously, AsyncResult maintained the order of notifications, but Event
@@ -30,6 +43,7 @@ class _AbstractLinkable(object):
         # uniqueness would be with a 2.7+ OrderedDict.)
         self._links = set()
         self.hub = get_hub()
+        self._notifier = None
 
     def ready(self):
         # Instances must define this
@@ -90,18 +104,17 @@ class _AbstractLinkable(object):
         # bool(self._notifier) would turn to False as soon as we exit this
         # method anyway.
         del todo
-        del self._notifier
+        self._notifier = None
 
     def _wait_core(self, timeout, catch=Timeout):
         # The core of the wait implementation, handling
         # switching and linking. If *catch* is set to (),
         # a timeout that elapses will be allowed to be raised.
         # Returns a true value if the wait succeeded without timing out.
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
-            timer = Timeout._start_new_or_dummy(timeout)
-            try:
+            with Timeout._start_new_or_dummy(timeout) as timer:
                 try:
                     result = self.hub.switch()
                     if result is not self: # pragma: no cover
@@ -113,8 +126,6 @@ class _AbstractLinkable(object):
                     # test_set_and_clear and test_timeout in test_threading
                     # rely on the exact return values, not just truthish-ness
                     return False
-            finally:
-                timer.cancel()
         finally:
             self.unlink(switch)
 
@@ -146,7 +157,11 @@ class Event(_AbstractLinkable):
         the waiting greenlets being awakened. These details may change in the future.
     """
 
-    _flag = False
+    __slots__ = ('_flag', '__weakref__')
+
+    def __init__(self):
+        _AbstractLinkable.__init__(self)
+        self._flag = False
 
     def __str__(self):
         return '<%s %s _links[%s]>' % (self.__class__.__name__, (self._flag and 'set') or 'clear', len(self._links))
@@ -155,8 +170,14 @@ class Event(_AbstractLinkable):
         """Return true if and only if the internal flag is true."""
         return self._flag
 
-    isSet = is_set  # makes it a better drop-in replacement for threading.Event
-    ready = is_set  # makes it compatible with AsyncResult and Greenlet (for example in wait())
+    def isSet(self):
+        # makes it a better drop-in replacement for threading.Event
+        return self._flag
+
+    def ready(self):
+        # makes it compatible with AsyncResult and Greenlet (for
+        # example in wait())
+        return self._flag
 
     def set(self):
         """
@@ -219,7 +240,7 @@ class Event(_AbstractLinkable):
         return self._wait(timeout)
 
     def _reset_internal_locks(self): # pragma: no cover
-        # for compatibility with threading.Event (only in case of patch_all(Event=True), by default Event is not patched)
+        # for compatibility with threading.Event
         #  Exception AttributeError: AttributeError("'Event' object has no attribute '_reset_internal_locks'",)
         # in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
         pass
@@ -275,9 +296,12 @@ class AsyncResult(_AbstractLinkable):
        merged.
     """
 
-    _value = _NONE
-    _exc_info = ()
-    _notifier = None
+    __slots__ = ('_value', '_exc_info', '_imap_task_index')
+
+    def __init__(self):
+        _AbstractLinkable.__init__(self)
+        self._value = _NONE
+        self._exc_info = ()
 
     @property
     def _exception(self):
@@ -357,7 +381,7 @@ class AsyncResult(_AbstractLinkable):
     def get(self, block=True, timeout=None):
         """Return the stored value or raise the exception.
 
-        If this instance already holds a value or an exception, return  or raise it immediatelly.
+        If this instance already holds a value or an exception, return  or raise it immediately.
         Otherwise, block until another greenlet calls :meth:`set` or :meth:`set_exception` or
         until the optional timeout occurs.
 
@@ -446,3 +470,12 @@ class AsyncResult(_AbstractLinkable):
         return False
 
     # exception is a method, we use it as a property
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent._event')

python/gevent/events.py

@@ -0,0 +1,480 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 gevent. See LICENSE for details.
+"""
+Publish/subscribe event infrastructure.
+
+When certain "interesting" things happen during the lifetime of the
+process, gevent will "publish" an event (an object). That event is
+delivered to interested "subscribers" (functions that take one
+parameter, the event object).
+
+Higher level frameworks may take this foundation and build richer
+models on it.
+
+If :mod:`zope.event` is installed, then it will be used to provide the
+functionality of `notify` and `subscribers`. See
+:mod:`zope.event.classhandler` for a simple class-based approach to
+subscribing to a filtered list of events, and see `zope.component
+<https://zopecomponent.readthedocs.io/en/latest/event.html>`_ for a
+much higher-level, flexible system. If you are using one of these systems,
+you generally will not want to directly modify `subscribers`.
+
+.. versionadded:: 1.3b1
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+__all__ = [
+    'subscribers',
+
+    # monitor thread
+    'IEventLoopBlocked',
+    'EventLoopBlocked',
+    'IMemoryUsageThresholdExceeded',
+    'MemoryUsageThresholdExceeded',
+    'IMemoryUsageUnderThreshold',
+    'MemoryUsageUnderThreshold',
+
+    # Hub
+    'IPeriodicMonitorThread',
+    'IPeriodicMonitorThreadStartedEvent',
+    'PeriodicMonitorThreadStartedEvent',
+
+    # monkey
+    'IGeventPatchEvent',
+    'GeventPatchEvent',
+    'IGeventWillPatchEvent',
+    'DoNotPatch',
+    'GeventWillPatchEvent',
+    'IGeventDidPatchEvent',
+    'IGeventWillPatchModuleEvent',
+    'GeventWillPatchModuleEvent',
+    'IGeventDidPatchModuleEvent',
+    'GeventDidPatchModuleEvent',
+    'IGeventWillPatchAllEvent',
+    'GeventWillPatchAllEvent',
+    'IGeventDidPatchBuiltinModulesEvent',
+    'GeventDidPatchBuiltinModulesEvent',
+    'IGeventDidPatchAllEvent',
+    'GeventDidPatchAllEvent',
+]
+
+# pylint:disable=no-self-argument
+
+try:
+    from zope.event import subscribers
+    from zope.event import notify
+except ImportError:
+    #: Applications may register for notification of events by appending a
+    #: callable to the ``subscribers`` list.
+    #:
+    #: Each subscriber takes a single argument, which is the event object
+    #: being published.
+    #:
+    #: Exceptions raised by subscribers will be propagated *without* running
+    #: any remaining subscribers.
+    subscribers = []
+
+    def notify(event):
+        """
+        Notify all subscribers of ``event``.
+        """
+        for subscriber in subscribers:
+            subscriber(event)
+
+notify = notify # export
+try:
+    # pkg_resources is technically optional, we don't
+    # list a hard dependency on it.
+    __import__('pkg_resources')
+except ImportError:
+    notify_and_call_entry_points = notify
+else:
+    from pkg_resources import iter_entry_points
+    import platform
+    try:
+        # Cache the platform info. pkg_resources uses
+        # platform.machine() for environment markers, and
+        # platform.machine() wants to call os.popen('uname'), which is
+        # broken on Py2 when the gevent child signal handler is
+        # installed. (see test__monkey_sigchild_2.py)
+        platform.uname()
+    except: # pylint:disable=bare-except
+        pass
+    finally:
+        del platform
+
+    def notify_and_call_entry_points(event):
+        notify(event)
+        for plugin in iter_entry_points(event.ENTRY_POINT_NAME):
+            subscriber = plugin.load()
+            subscriber(event)
+
+from gevent._util import Interface
+from gevent._util import implementer
+from gevent._util import Attribute
+
+
+class IPeriodicMonitorThread(Interface):
+    """
+    The contract for the periodic monitoring thread that is started
+    by the hub.
+    """
+
+    def add_monitoring_function(function, period):
+        """
+        Schedule the *function* to be called approximately every *period* fractional seconds.
+
+        The *function* receives one argument, the hub being monitored. It is called
+        in the monitoring thread, *not* the hub thread. It **must not** attempt to
+        use the gevent asynchronous API.
+
+        If the *function* is already a monitoring function, then its *period*
+        will be updated for future runs.
+
+        If the *period* is ``None``, then the function will be removed.
+
+        A *period* less than or equal to zero is not allowed.
+        """
+
+class IPeriodicMonitorThreadStartedEvent(Interface):
+    """
+    The event emitted when a hub starts a periodic monitoring thread.
+
+    You can use this event to add additional monitoring functions.
+    """
+
+    monitor = Attribute("The instance of `IPeriodicMonitorThread` that was started.")
+
+class PeriodicMonitorThreadStartedEvent(object):
+    """
+    The implementation of :class:`IPeriodicMonitorThreadStartedEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.hub.periodic_monitor_thread_started'
+
+    def __init__(self, monitor):
+        self.monitor = monitor
+
+class IEventLoopBlocked(Interface):
+    """
+    The event emitted when the event loop is blocked.
+
+    This event is emitted in the monitor thread.
+    """
+
+    greenlet = Attribute("The greenlet that appeared to be blocking the loop.")
+    blocking_time = Attribute("The approximate time in seconds the loop has been blocked.")
+    info = Attribute("A sequence of string lines providing extra info.")
+
+@implementer(IEventLoopBlocked)
+class EventLoopBlocked(object):
+    """
+    The event emitted when the event loop is blocked.
+
+    Implements `IEventLoopBlocked`.
+    """
+
+    def __init__(self, greenlet, blocking_time, info):
+        self.greenlet = greenlet
+        self.blocking_time = blocking_time
+        self.info = info
+
+class IMemoryUsageThresholdExceeded(Interface):
+    """
+    The event emitted when the memory usage threshold is exceeded.
+
+    This event is emitted only while memory continues to grow
+    above the threshold. Only if the condition or stabilized is corrected (memory
+    usage drops) will the event be emitted in the future.
+
+    This event is emitted in the monitor thread.
+    """
+
+    mem_usage = Attribute("The current process memory usage, in bytes.")
+    max_allowed = Attribute("The maximum allowed memory usage, in bytes.")
+    memory_info = Attribute("The tuple of memory usage stats return by psutil.")
+
+class _AbstractMemoryEvent(object):
+
+    def __init__(self, mem_usage, max_allowed, memory_info):
+        self.mem_usage = mem_usage
+        self.max_allowed = max_allowed
+        self.memory_info = memory_info
+
+    def __repr__(self):
+        return "<%s used=%d max=%d details=%r>" % (
+            self.__class__.__name__,
+            self.mem_usage,
+            self.max_allowed,
+            self.memory_info,
+        )
+
+@implementer(IMemoryUsageThresholdExceeded)
+class MemoryUsageThresholdExceeded(_AbstractMemoryEvent):
+    """
+    Implementation of `IMemoryUsageThresholdExceeded`.
+    """
+
+
+class IMemoryUsageUnderThreshold(Interface):
+    """
+    The event emitted when the memory usage drops below the
+    threshold after having previously been above it.
+
+    This event is emitted only the first time memory usage is detected
+    to be below the threshold after having previously been above it.
+    If memory usage climbs again, a `IMemoryUsageThresholdExceeded`
+    event will be broadcast, and then this event could be broadcast again.
+
+    This event is emitted in the monitor thread.
+    """
+
+    mem_usage = Attribute("The current process memory usage, in bytes.")
+    max_allowed = Attribute("The maximum allowed memory usage, in bytes.")
+    max_memory_usage = Attribute("The memory usage that caused the previous "
+                                 "IMemoryUsageThresholdExceeded event.")
+    memory_info = Attribute("The tuple of memory usage stats return by psutil.")
+
+
+@implementer(IMemoryUsageUnderThreshold)
+class MemoryUsageUnderThreshold(_AbstractMemoryEvent):
+    """
+    Implementation of `IMemoryUsageUnderThreshold`.
+    """
+
+    def __init__(self, mem_usage, max_allowed, memory_info, max_usage):
+        super(MemoryUsageUnderThreshold, self).__init__(mem_usage, max_allowed, memory_info)
+        self.max_memory_usage = max_usage
+
+
+class IGeventPatchEvent(Interface):
+    """
+    The root for all monkey-patch events gevent emits.
+    """
+
+    source = Attribute("The source object containing the patches.")
+    target = Attribute("The destination object to be patched.")
+
+@implementer(IGeventPatchEvent)
+class GeventPatchEvent(object):
+    """
+    Implementation of `IGeventPatchEvent`.
+    """
+
+    def __init__(self, source, target):
+        self.source = source
+        self.target = target
+
+    def __repr__(self):
+        return '<%s source=%r target=%r at %x>' % (self.__class__.__name__,
+                                                   self.source,
+                                                   self.target,
+                                                   id(self))
+
+class IGeventWillPatchEvent(IGeventPatchEvent):
+    """
+    An event emitted *before* gevent monkey-patches something.
+
+    If a subscriber raises `DoNotPatch`, then patching this particular
+    item will not take place.
+    """
+
+
+class DoNotPatch(BaseException):
+    """
+    Subscribers to will-patch events can raise instances
+    of this class to tell gevent not to patch that particular item.
+    """
+
+
+@implementer(IGeventWillPatchEvent)
+class GeventWillPatchEvent(GeventPatchEvent):
+    """
+    Implementation of `IGeventWillPatchEvent`.
+    """
+
+class IGeventDidPatchEvent(IGeventPatchEvent):
+    """
+    An event emitted *after* gevent has patched something.
+    """
+
+@implementer(IGeventDidPatchEvent)
+class GeventDidPatchEvent(GeventPatchEvent):
+    """
+    Implementation of `IGeventDidPatchEvent`.
+    """
+
+class IGeventWillPatchModuleEvent(IGeventWillPatchEvent):
+    """
+    An event emitted *before* gevent begins patching a specific module.
+
+    Both *source* and *target* attributes are module objects.
+    """
+
+    module_name = Attribute("The name of the module being patched. "
+                            "This is the same as ``target.__name__``.")
+
+    target_item_names = Attribute("The list of item names to patch. "
+                                  "This can be modified in place with caution.")
+
+@implementer(IGeventWillPatchModuleEvent)
+class GeventWillPatchModuleEvent(GeventWillPatchEvent):
+    """
+    Implementation of `IGeventWillPatchModuleEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.monkey.will_patch_module'
+
+    def __init__(self, module_name, source, target, items):
+        super(GeventWillPatchModuleEvent, self).__init__(source, target)
+        self.module_name = module_name
+        self.target_item_names = items
+
+
+class IGeventDidPatchModuleEvent(IGeventDidPatchEvent):
+    """
+    An event emitted *after* gevent has completed patching a specific
+    module.
+    """
+
+    module_name = Attribute("The name of the module being patched. "
+                            "This is the same as ``target.__name__``.")
+
+
+@implementer(IGeventDidPatchModuleEvent)
+class GeventDidPatchModuleEvent(GeventDidPatchEvent):
+    """
+    Implementation of `IGeventDidPatchModuleEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_module'
+
+    def __init__(self, module_name, source, target):
+        super(GeventDidPatchModuleEvent, self).__init__(source, target)
+        self.module_name = module_name
+
+# TODO: Maybe it would be useful for the the module patch events
+# to have an attribute telling if they're being done during patch_all?
+
+class IGeventWillPatchAllEvent(IGeventWillPatchEvent):
+    """
+    An event emitted *before* gevent begins patching the system.
+
+    Following this event will be a series of
+    `IGeventWillPatchModuleEvent` and `IGeventDidPatchModuleEvent` for
+    each patched module.
+
+    Once the gevent builtin modules have been processed,
+    `IGeventDidPatchBuiltinModulesEvent` will be emitted. Processing
+    this event is an ideal time for third-party modules to be imported
+    and patched (which may trigger its own will/did patch module
+    events).
+
+    Finally, a `IGeventDidPatchAllEvent` will be sent.
+
+    If a subscriber to this event raises `DoNotPatch`, no patching
+    will be done.
+
+    The *source* and *target* attributes have undefined values.
+    """
+
+    patch_all_arguments = Attribute(
+        "A dictionary of all the arguments to `gevent.monkey.patch_all`. "
+        "This dictionary should not be modified. "
+    )
+
+    patch_all_kwargs = Attribute(
+        "A dictionary of the extra arguments to `gevent.monkey.patch_all`. "
+        "This dictionary should not be modified. "
+    )
+
+    def will_patch_module(module_name):
+        """
+        Return whether the module named *module_name* will be patched.
+        """
+
+class _PatchAllMixin(object):
+    def __init__(self, patch_all_arguments, patch_all_kwargs):
+        super(_PatchAllMixin, self).__init__(None, None)
+        self._patch_all_arguments = patch_all_arguments
+        self._patch_all_kwargs = patch_all_kwargs
+
+    @property
+    def patch_all_arguments(self):
+        return self._patch_all_arguments.copy()
+
+    @property
+    def patch_all_kwargs(self):
+        return self._patch_all_kwargs.copy()
+
+    def __repr__(self):
+        return '<%s %r at %x>' % (self.__class__.__name__,
+                                  self._patch_all_arguments,
+                                  id(self))
+
+@implementer(IGeventWillPatchAllEvent)
+class GeventWillPatchAllEvent(_PatchAllMixin, GeventWillPatchEvent):
+    """
+    Implementation of `IGeventWillPatchAllEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.monkey.will_patch_all'
+
+    def will_patch_module(self, module_name):
+        return self.patch_all_arguments.get(module_name)
+
+class IGeventDidPatchBuiltinModulesEvent(IGeventDidPatchEvent):
+    """
+    Event emitted *after* the builtin modules have been patched.
+
+    The values of the *source* and *target* attributes are undefined.
+    """
+
+    patch_all_arguments = Attribute(
+        "A dictionary of all the arguments to `gevent.monkey.patch_all`. "
+        "This dictionary should not be modified. "
+    )
+
+    patch_all_kwargs = Attribute(
+        "A dictionary of the extra arguments to `gevent.monkey.patch_all`. "
+        "This dictionary should not be modified. "
+    )
+
+@implementer(IGeventDidPatchBuiltinModulesEvent)
+class GeventDidPatchBuiltinModulesEvent(_PatchAllMixin, GeventDidPatchEvent):
+    """
+    Implementation of `IGeventDidPatchBuiltinModulesEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_builtins'
+
+class IGeventDidPatchAllEvent(IGeventDidPatchEvent):
+    """
+    Event emitted after gevent has patched all modules, both builtin
+    and those provided by plugins/subscribers.
+
+    The values of the *source* and *target* attributes are undefined.
+    """
+
+@implementer(IGeventDidPatchAllEvent)
+class GeventDidPatchAllEvent(_PatchAllMixin, GeventDidPatchEvent):
+    """
+    Implementation of `IGeventDidPatchAllEvent`.
+    """
+
+    #: The name of the setuptools entry point that is called when this
+    #: event is emitted.
+    ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_all'

python/gevent/exceptions.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# copyright 2018 gevent
+"""
+Exceptions.
+
+.. versionadded:: 1.3b1
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+__all__ = [
+    'LoopExit',
+]
+
+
+class LoopExit(Exception):
+    """
+    Exception thrown when the hub finishes running (`gevent.hub.Hub.run`
+    would return).
+
+    In a normal application, this is never thrown or caught
+    explicitly. The internal implementation of functions like
+    :meth:`gevent.hub.Hub.join` and :func:`gevent.joinall` may catch it, but user code
+    generally should not.
+
+    .. caution::
+       Errors in application programming can also lead to this exception being
+       raised. Some examples include (but are not limited too):
+
+       - greenlets deadlocking on a lock;
+       - using a socket or other gevent object with native thread
+         affinity from a different thread
+
+    """
+
+    def __repr__(self):
+        # pylint:disable=unsubscriptable-object
+        if len(self.args) == 3: # From the hub
+            import pprint
+            return "%s\n\tHub: %s\n\tHandles:\n%s" % (
+                self.args[0], self.args[1],
+                pprint.pformat(self.args[2])
+            )
+        return Exception.__repr__(self)
+
+    def __str__(self):
+        return repr(self)
+
+class BlockingSwitchOutError(AssertionError):
+    """
+    Raised when a gevent synchronous function is called from a
+    low-level event loop callback.
+
+    This is usually a programming error.
+    """
+
+
+class InvalidSwitchError(AssertionError):
+    """
+    Raised when the event loop returns control to a greenlet in an
+    unexpected way.
+
+    This is usually a bug in gevent, greenlet, or the event loop.
+    """
+
+class ConcurrentObjectUseError(AssertionError):
+    """
+    Raised when an object is used (waited on) by two greenlets
+    independently, meaning the object was entered into a blocking
+    state by one greenlet and then another while still blocking in the
+    first one.
+
+    This is usually a programming error.
+
+    .. seealso:: `gevent.socket.wait`
+    """

python/gevent/fileobject.py

@@ -35,31 +35,12 @@ Classes
 """
 from __future__ import absolute_import
 
-import functools
-import sys
-import os
-
-from gevent._fileobjectcommon import FileObjectClosed
-from gevent._fileobjectcommon import FileObjectBase
-from gevent.hub import get_hub
-from gevent._compat import integer_types
-from gevent._compat import reraise
-from gevent.lock import Semaphore, DummySemaphore
-
-
-PYPY = hasattr(sys, 'pypy_version_info')
-
-if hasattr(sys, 'exc_clear'):
-    def _exc_clear():
-        sys.exc_clear()
-else:
-    def _exc_clear():
-        return
-
+from gevent._config import config
 
 __all__ = [
     'FileObjectPosix',
     'FileObjectThread',
+    'FileObjectBlock',
     'FileObject',
 ]
 
@@ -71,149 +52,10 @@ else:
     del fcntl
     from gevent._fileobjectposix import FileObjectPosix
 
-
-class FileObjectThread(FileObjectBase):
-    """
-    A file-like object wrapping another file-like object, performing all blocking
-    operations on that object in a background thread.
-
-    .. caution::
-        Attempting to change the threadpool or lock of an existing FileObjectThread
-        has undefined consequences.
-
-    .. versionchanged:: 1.1b1
-       The file object is closed using the threadpool. Note that whether or
-       not this action is synchronous or asynchronous is not documented.
-
-    """
-
-    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
-        """
-        :param fobj: The underlying file-like object to wrap, or an integer fileno
-           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
-        :keyword bool lock: If True (the default) then all operations will
-           be performed one-by-one. Note that this does not guarantee that, if using
-           this file object from multiple threads/greenlets, operations will be performed
-           in any particular order, only that no two operations will be attempted at the
-           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
-           file operations with an external resource.
-        :keyword bool close: If True (the default) then when this object is closed,
-           the underlying object is closed as well.
-        """
-        closefd = close
-        self.threadpool = threadpool or get_hub().threadpool
-        self.lock = lock
-        if self.lock is True:
-            self.lock = Semaphore()
-        elif not self.lock:
-            self.lock = DummySemaphore()
-        if not hasattr(self.lock, '__enter__'):
-            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectThread does not support close=False on an fd.')
-            if mode is None:
-                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
-                fobj = os.fdopen(fobj)
-            else:
-                fobj = os.fdopen(fobj, mode, bufsize)
-
-        self.__io_holder = [fobj] # signal for _wrap_method
-        super(FileObjectThread, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        self.__io_holder[0] = None # for _wrap_method
-        try:
-            with self.lock:
-                self.threadpool.apply(fobj.flush)
-        finally:
-            if closefd:
-                # Note that we're not taking the lock; older code
-                # did fobj.close() without going through the threadpool at all,
-                # so acquiring the lock could potentially introduce deadlocks
-                # that weren't present before. Avoiding the lock doesn't make
-                # the existing race condition any worse.
-                # We wrap the close in an exception handler and re-raise directly
-                # to avoid the (common, expected) IOError from being logged by the pool
-                def close():
-                    try:
-                        fobj.close()
-                    except: # pylint:disable=bare-except
-                        return sys.exc_info()
-                exc_info = self.threadpool.apply(close)
-                if exc_info:
-                    reraise(*exc_info)
-
-    def _do_delegate_methods(self):
-        super(FileObjectThread, self)._do_delegate_methods()
-        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
-            self.read1 = self.read
-        self.__io_holder[0] = self._io
-
-    def _extra_repr(self):
-        return ' threadpool=%r' % (self.threadpool,)
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        line = self.readline()
-        if line:
-            return line
-        raise StopIteration
-    __next__ = next
-
-    def _wrap_method(self, method):
-        # NOTE: We are careful to avoid introducing a refcycle
-        # within self. Our wrapper cannot refer to self.
-        io_holder = self.__io_holder
-        lock = self.lock
-        threadpool = self.threadpool
-
-        @functools.wraps(method)
-        def thread_method(*args, **kwargs):
-            if io_holder[0] is None:
-                # This is different than FileObjectPosix, etc,
-                # because we want to save the expensive trip through
-                # the threadpool.
-                raise FileObjectClosed()
-            with lock:
-                return threadpool.apply(method, args, kwargs)
-
-        return thread_method
+from gevent._fileobjectcommon import FileObjectThread
+from gevent._fileobjectcommon import FileObjectBlock
 
 
-try:
-    FileObject = FileObjectPosix
-except NameError:
-    FileObject = FileObjectThread
-
-
-class FileObjectBlock(FileObjectBase):
-
-    def __init__(self, fobj, *args, **kwargs):
-        closefd = kwargs.pop('close', True)
-        if kwargs:
-            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectBlock does not support close=False on an fd.')
-            fobj = os.fdopen(fobj, *args)
-        super(FileObjectBlock, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        fobj.close()
-
-config = os.environ.get('GEVENT_FILE')
-if config:
-    klass = {'thread': 'gevent.fileobject.FileObjectThread',
-             'posix': 'gevent.fileobject.FileObjectPosix',
-             'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
-    if klass.startswith('gevent.fileobject.'):
-        FileObject = globals()[klass.split('.', 2)[-1]]
-    else:
-        from gevent.hub import _import
-        FileObject = _import(klass)
-    del klass
+# None of the possible objects can live in this module because
+# we would get an import cycle and the config couldn't be set from code.
+FileObject = config.fileobject

python/gevent/gevent.ares.h

@@ -1,48 +0,0 @@
-/* Generated by Cython 0.25.2 */
-
-#ifndef __PYX_HAVE__gevent__ares
-#define __PYX_HAVE__gevent__ares
-
-struct PyGeventAresChannelObject;
-
-/* "gevent/ares.pyx":245
- * 
- * 
- * cdef public class channel [object PyGeventAresChannelObject, type PyGeventAresChannel_Type]:             # <<<<<<<<<<<<<<
- * 
- *     cdef public object loop
- */
-struct PyGeventAresChannelObject {
-  PyObject_HEAD
-  struct __pyx_vtabstruct_6gevent_4ares_channel *__pyx_vtab;
-  PyObject *loop;
-  struct ares_channeldata *channel;
-  PyObject *_watchers;
-  PyObject *_timer;
-};
-
-#ifndef __PYX_HAVE_API__gevent__ares
-
-#ifndef __PYX_EXTERN_C
-  #ifdef __cplusplus
-    #define __PYX_EXTERN_C extern "C"
-  #else
-    #define __PYX_EXTERN_C extern
-  #endif
-#endif
-
-#ifndef DL_IMPORT
-  #define DL_IMPORT(_T) _T
-#endif
-
-__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventAresChannel_Type;
-
-#endif /* !__PYX_HAVE_API__gevent__ares */
-
-#if PY_MAJOR_VERSION < 3
-PyMODINIT_FUNC initares(void);
-#else
-PyMODINIT_FUNC PyInit_ares(void);
-#endif
-
-#endif /* !__PYX_HAVE__gevent__ares */

python/gevent/greenlet.py

@@ -1,22 +1,34 @@
 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
-from __future__ import absolute_import
-import sys
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+
+from __future__ import absolute_import, print_function, division
+
+from sys import _getframe as sys_getframe
+from sys import exc_info as sys_exc_info
+from weakref import ref as wref
+
+# XXX: How to get cython to let us rename this as RawGreenlet
+# like we prefer?
 from greenlet import greenlet
-from gevent._compat import PY3
-from gevent._compat import PYPY
+from greenlet import GreenletExit
+
 from gevent._compat import reraise
-from gevent._util import Lazy
+from gevent._compat import PYPY as _PYPY
 from gevent._tblib import dump_traceback
 from gevent._tblib import load_traceback
-from gevent.hub import GreenletExit
-from gevent.hub import InvalidSwitchError
-from gevent.hub import Waiter
-from gevent.hub import get_hub
-from gevent.hub import getcurrent
-from gevent.hub import iwait
-from gevent.hub import wait
+
+from gevent.exceptions import InvalidSwitchError
+
+from gevent._hub_primitives import iwait_on_objects as iwait
+from gevent._hub_primitives import wait_on_objects as wait
+
 from gevent.timeout import Timeout
-from collections import deque
+
+from gevent._config import config as GEVENT_CONFIG
+from gevent._util import Lazy
+from gevent._util import readproperty
+from gevent._hub_local import get_hub_noargs as get_hub
+from gevent import _waiter
 
 
 __all__ = [
@@ -26,13 +38,23 @@ __all__ = [
 ]
 
 
-if PYPY:
+# In Cython, we define these as 'cdef inline' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+locals()['Waiter'] = _waiter.Waiter
+
+
+if _PYPY:
     import _continuation # pylint:disable=import-error
     _continulet = _continuation.continulet
 
 
 class SpawnedLink(object):
-    """A wrapper around link that calls it in another greenlet.
+    """
+    A wrapper around link that calls it in another greenlet.
 
     Can be called only from main loop.
     """
@@ -87,39 +109,80 @@ class FailureSpawnedLink(SpawnedLink):
         if not source.successful():
             return SpawnedLink.__call__(self, source)
 
+class _Frame(object):
+
+    __slots__ = ('f_code', 'f_lineno', 'f_back')
+
+    def __init__(self, f_code, f_lineno, f_back):
+        self.f_code = f_code
+        self.f_lineno = f_lineno
+        self.f_back = f_back
+
+    @property
+    def f_globals(self):
+        return None
+
+def _Frame_from_list(frames):
+    previous = None
+    for frame in reversed(frames):
+        f = _Frame(frame[0], frame[1], previous)
+        previous = f
+    return previous
+
+def _extract_stack(limit):
+    try:
+        frame = sys_getframe()
+    except ValueError:
+        # In certain embedded cases that directly use the Python C api
+        # to call Greenlet.spawn (e.g., uwsgi) this can raise
+        # `ValueError: call stack is not deep enough`. This is because
+        # the Cython stack frames for Greenlet.spawn ->
+        # Greenlet.__init__ -> _extract_stack are all on the C level,
+        # not the Python level.
+        # See https://github.com/gevent/gevent/issues/1212
+        frame = None
+
+    frames = []
+
+    while limit and frame is not None:
+        limit -= 1
+        frames.append((frame.f_code, frame.f_lineno))
+        frame = frame.f_back
+
+    return frames
+
+
+_greenlet__init__ = greenlet.__init__
+
 class Greenlet(greenlet):
-    """A light-weight cooperatively-scheduled execution unit.
+    """
+    A light-weight cooperatively-scheduled execution unit.
     """
     # pylint:disable=too-many-public-methods,too-many-instance-attributes
 
-    value = None
-    _exc_info = ()
-    _notifier = None
-
-    #: An event, such as a timer or a callback that fires. It is established in
-    #: start() and start_later() as those two objects, respectively.
-    #: Once this becomes non-None, the Greenlet cannot be started again. Conversely,
-    #: kill() and throw() check for non-None to determine if this object has ever been
-    #: scheduled for starting. A placeholder _dummy_event is assigned by them to prevent
-    #: the greenlet from being started in the future, if necessary.
-    _start_event = None
-    args = ()
-    _kwargs = None
+    spawning_stack_limit = 10
 
+    # pylint:disable=keyword-arg-before-vararg,super-init-not-called
     def __init__(self, run=None, *args, **kwargs):
         """
-        Greenlet constructor.
-
         :param args: The arguments passed to the ``run`` function.
         :param kwargs: The keyword arguments passed to the ``run`` function.
-        :keyword run: The callable object to run. If not given, this object's
+        :keyword callable run: The callable object to run. If not given, this object's
             `_run` method will be invoked (typically defined by subclasses).
 
         .. versionchanged:: 1.1b1
             The ``run`` argument to the constructor is now verified to be a callable
             object. Previously, passing a non-callable object would fail after the greenlet
             was spawned.
+
+        .. versionchanged:: 1.3b1
+           The ``GEVENT_TRACK_GREENLET_TREE`` configuration value may be set to
+           a false value to disable ``spawn_tree_locals``, ``spawning_greenlet``,
+           and ``spawning_stack``. The first two will be None in that case, and the
+           latter will be empty.
         """
+        # The attributes are documented in the .rst file
+
         # greenlet.greenlet(run=None, parent=None)
         # Calling it with both positional arguments instead of a keyword
         # argument (parent=get_hub()) speeds up creation of this object ~30%:
@@ -128,7 +191,26 @@ class Greenlet(greenlet):
         # Python 3.4: 2.32usec with keywords vs 1.74usec with positional
         # Python 3.3: 2.55usec with keywords vs 1.92usec with positional
         # Python 2.7: 1.73usec with keywords vs 1.40usec with positional
-        greenlet.__init__(self, None, get_hub())
+
+        # Timings taken Feb 21 2018 prior to integration of #755
+        # python -m perf timeit -s 'import gevent' 'gevent.Greenlet()'
+        # 3.6.4       : Mean +- std dev: 1.08 us +- 0.05 us
+        # 2.7.14      : Mean +- std dev: 1.44 us +- 0.06 us
+        # PyPy2 5.10.0: Mean +- std dev: 2.14 ns +- 0.08 ns
+
+        # After the integration of spawning_stack, spawning_greenlet,
+        # and spawn_tree_locals on that same date:
+        # 3.6.4       : Mean +- std dev: 8.92 us +- 0.36 us ->  8.2x
+        # 2.7.14      : Mean +- std dev: 14.8 us +- 0.5 us  -> 10.2x
+        # PyPy2 5.10.0: Mean +- std dev: 3.24 us +- 0.17 us ->  1.5x
+
+        # Compiling with Cython gets us to these numbers:
+        # 3.6.4        : Mean +- std dev: 3.63 us +- 0.14 us
+        # 2.7.14       : Mean +- std dev: 3.37 us +- 0.20 us
+        # PyPy2 5.10.0 : Mean +- std dev: 4.44 us +- 0.28 us
+
+
+        _greenlet__init__(self, None, get_hub())
 
         if run is not None:
             self._run = run
@@ -139,21 +221,92 @@ class Greenlet(greenlet):
         if not callable(self._run):
             raise TypeError("The run argument or self._run must be callable")
 
-        if args:
-            self.args = args
-        if kwargs:
-            self._kwargs = kwargs
+        self.args = args
+        self.kwargs = kwargs
+        self.value = None
 
-    @property
-    def kwargs(self):
-        return self._kwargs or {}
+        #: An event, such as a timer or a callback that fires. It is established in
+        #: start() and start_later() as those two objects, respectively.
+        #: Once this becomes non-None, the Greenlet cannot be started again. Conversely,
+        #: kill() and throw() check for non-None to determine if this object has ever been
+        #: scheduled for starting. A placeholder _dummy_event is assigned by them to prevent
+        #: the greenlet from being started in the future, if necessary.
+        self._start_event = None
+
+        self._notifier = None
+        self._formatted_info = None
+        self._links = []
+        self._ident = None
+
+        # Initial state: None.
+        # Completed successfully: (None, None, None)
+        # Failed with exception: (t, v, dump_traceback(tb)))
+        self._exc_info = None
+
+        if GEVENT_CONFIG.track_greenlet_tree:
+            spawner = getcurrent() # pylint:disable=undefined-variable
+            self.spawning_greenlet = wref(spawner)
+            try:
+                self.spawn_tree_locals = spawner.spawn_tree_locals
+            except AttributeError:
+                self.spawn_tree_locals = {}
+                if spawner.parent is not None:
+                    # The main greenlet has no parent.
+                    # Its children get separate locals.
+                    spawner.spawn_tree_locals = self.spawn_tree_locals
+
+            self._spawning_stack_frames = _extract_stack(self.spawning_stack_limit)
+            self._spawning_stack_frames.extend(getattr(spawner, '_spawning_stack_frames', []))
+        else:
+            # None is the default for all of these in Cython, but we
+            # need to declare them for pure-Python mode.
+            self.spawning_greenlet = None
+            self.spawn_tree_locals = None
+            self._spawning_stack_frames = None
 
     @Lazy
-    def _links(self):
-        return deque()
+    def spawning_stack(self):
+        # Store this in the __dict__. We don't use it from the C
+        # code. It's tempting to discard _spawning_stack_frames
+        # after this, but child greenlets may still be created
+        # that need it.
+        return _Frame_from_list(self._spawning_stack_frames or [])
 
-    def _has_links(self):
-        return '_links' in self.__dict__ and self._links
+    def _get_minimal_ident(self):
+        reg = self.parent.ident_registry
+        return reg.get_ident(self)
+
+    @property
+    def minimal_ident(self):
+        """
+        A small, unique integer that identifies this object.
+
+        This is similar to :attr:`threading.Thread.ident` (and `id`)
+        in that as long as this object is alive, no other greenlet *in
+        this hub* will have the same id, but it makes a stronger
+        guarantee that the assigned values will be small and
+        sequential. Sometime after this object has died, the value
+        will be available for reuse.
+
+        To get ids that are unique across all hubs, combine this with
+        the hub's ``minimal_ident``.
+
+        .. versionadded:: 1.3a2
+        """
+        if self._ident is None:
+            self._ident = self._get_minimal_ident()
+        return self._ident
+
+    @readproperty
+    def name(self):
+        """
+        The greenlet name. By default, a unique name is constructed using
+        the :attr:`minimal_ident`. You can assign a string to this
+        value to change it. It is shown in the `repr` of this object.
+
+        .. versionadded:: 1.3a2
+        """
+        return 'Greenlet-%d' % (self.minimal_ident)
 
     def _raise_exception(self):
         reraise(*self.exc_info)
@@ -163,50 +316,52 @@ class Greenlet(greenlet):
         # needed by killall
         return self.parent.loop
 
-    def __bool__(self):
-        return self._start_event is not None and self._exc_info is Greenlet._exc_info
-    __nonzero__ = __bool__
+    def __nonzero__(self):
+        return self._start_event is not None and self._exc_info is None
+    try:
+        __bool__ = __nonzero__ # Python 3
+    except NameError: # pragma: no cover
+        # When we're compiled with Cython, the __nonzero__ function
+        # goes directly into the slot and can't be accessed by name.
+        pass
 
     ### Lifecycle
 
-    if PYPY:
+    if _PYPY:
         # oops - pypy's .dead relies on __nonzero__ which we overriden above
         @property
         def dead(self):
+            "Boolean indicating that the greenlet is dead and will not run again."
             if self._greenlet__main:
                 return False
-            if self.__start_cancelled_by_kill or self.__started_but_aborted:
+            if self.__start_cancelled_by_kill() or self.__started_but_aborted():
                 return True
 
             return self._greenlet__started and not _continulet.is_pending(self)
     else:
         @property
         def dead(self):
-            return self.__start_cancelled_by_kill or self.__started_but_aborted or greenlet.dead.__get__(self)
+            "Boolean indicating that the greenlet is dead and will not run again."
+            return self.__start_cancelled_by_kill() or self.__started_but_aborted() or greenlet.dead.__get__(self)
 
-    @property
     def __never_started_or_killed(self):
         return self._start_event is None
 
-    @property
     def __start_pending(self):
         return (self._start_event is not None
                 and (self._start_event.pending or getattr(self._start_event, 'active', False)))
 
-    @property
     def __start_cancelled_by_kill(self):
         return self._start_event is _cancelled_start_event
 
-    @property
     def __start_completed(self):
         return self._start_event is _start_completed_event
 
-    @property
     def __started_but_aborted(self):
-        return (not self.__never_started_or_killed # we have been started or killed
-                and not self.__start_cancelled_by_kill # we weren't killed, so we must have been started
-                and not self.__start_completed # the start never completed
-                and not self.__start_pending) # and we're not pending, so we must have been aborted
+        return (not self.__never_started_or_killed() # we have been started or killed
+                and not self.__start_cancelled_by_kill() # we weren't killed, so we must have been started
+                and not self.__start_completed() # the start never completed
+                and not self.__start_pending()) # and we're not pending, so we must have been aborted
 
     def __cancel_start(self):
         if self._start_event is None:
@@ -219,10 +374,11 @@ class Greenlet(greenlet):
         # variable copy of that list (in _run_callbacks). This isn't a problem,
         # except for the leak-tests.
         self._start_event.stop()
+        self._start_event.close()
 
-    def __handle_death_before_start(self, *args):
+    def __handle_death_before_start(self, args):
         # args is (t, v, tb) or simply t or v
-        if self._exc_info is Greenlet._exc_info and self.dead:
+        if self._exc_info is None and self.dead:
             # the greenlet was never switched to before and it will never be, _report_error was not called
             # the result was not set and the links weren't notified. let's do it here.
             # checking that self.dead is true is essential, because throw() does not necessarily kill the greenlet
@@ -252,7 +408,7 @@ class Greenlet(greenlet):
             This function is only guaranteed to return true or false *values*, not
             necessarily the literal constants ``True`` or ``False``.
         """
-        return self.dead or self._exc_info
+        return self.dead or self._exc_info is not None
 
     def successful(self):
         """
@@ -266,34 +422,39 @@ class Greenlet(greenlet):
         .. note:: This function is only guaranteed to return true or false *values*,
               not necessarily the literal constants ``True`` or ``False``.
         """
-        return self._exc_info and self._exc_info[1] is None
+        return self._exc_info is not None and self._exc_info[1] is None
 
     def __repr__(self):
         classname = self.__class__.__name__
-        result = '<%s at %s' % (classname, hex(id(self)))
+        result = '<%s "%s" at %s' % (classname, self.name, hex(id(self)))
         formatted = self._formatinfo()
         if formatted:
             result += ': ' + formatted
         return result + '>'
 
-    _formatted_info = None
 
     def _formatinfo(self):
         info = self._formatted_info
         if info is not None:
             return info
 
-        try:
-            result = getfuncname(self.__dict__['_run'])
-        except Exception: # pylint:disable=broad-except
-            # Don't cache
-            return ''
+        # Are we running an arbitrary function provided to the constructor,
+        # or did a subclass override _run?
+        func = self._run
+        im_self = getattr(func, '__self__', None)
+        if im_self is self:
+            funcname = '_run'
+        elif im_self is not None:
+            funcname = repr(func)
+        else:
+            funcname = getattr(func, '__name__', '') or repr(func)
 
+        result = funcname
         args = []
         if self.args:
             args = [repr(x)[:50] for x in self.args]
-        if self._kwargs:
-            args.extend(['%s=%s' % (key, repr(value)[:50]) for (key, value) in self._kwargs.items()])
+        if self.kwargs:
+            args.extend(['%s=%s' % (key, repr(value)[:50]) for (key, value) in self.kwargs.items()])
         if args:
             result += '(' + ', '.join(args) + ')'
         # it is important to save the result here, because once the greenlet exits '_run' attribute will be removed
@@ -302,10 +463,11 @@ class Greenlet(greenlet):
 
     @property
     def exception(self):
-        """Holds the exception instance raised by the function if the greenlet has finished with an error.
-        Otherwise ``None``.
         """
-        return self._exc_info[1] if self._exc_info else None
+        Holds the exception instance raised by the function if the
+        greenlet has finished with an error. Otherwise ``None``.
+        """
+        return self._exc_info[1] if self._exc_info is not None else None
 
     @property
     def exc_info(self):
@@ -317,12 +479,12 @@ class Greenlet(greenlet):
 
         .. versionadded:: 1.1
         """
-        e = self._exc_info
-        if e and e[0] is not None:
-            return (e[0], e[1], load_traceback(e[2]))
+        ei = self._exc_info
+        if ei is not None and ei[0] is not None:
+            return (ei[0], ei[1], load_traceback(ei[2]))
 
     def throw(self, *args):
-        """Immediatelly switch into the greenlet and raise an exception in it.
+        """Immediately switch into the greenlet and raise an exception in it.
 
         Should only be called from the HUB, otherwise the current greenlet is left unscheduled forever.
         To raise an exception in a safe manner from any greenlet, use :meth:`kill`.
@@ -341,7 +503,7 @@ class Greenlet(greenlet):
                 # LoopExit.
                 greenlet.throw(self, *args)
         finally:
-            self.__handle_death_before_start(*args)
+            self.__handle_death_before_start(args)
 
     def start(self):
         """Schedule the greenlet to run in this loop iteration"""
@@ -349,7 +511,12 @@ class Greenlet(greenlet):
             self._start_event = self.parent.loop.run_callback(self.switch)
 
     def start_later(self, seconds):
-        """Schedule the greenlet to run in the future loop iteration *seconds* later"""
+        """
+        start_later(seconds) -> None
+
+        Schedule the greenlet to run in the future loop iteration
+        *seconds* later
+        """
         if self._start_event is None:
             self._start_event = self.parent.loop.timer(seconds)
             self._start_event.start(self.switch)
@@ -357,6 +524,8 @@ class Greenlet(greenlet):
     @classmethod
     def spawn(cls, *args, **kwargs):
         """
+        spawn(function, *args, **kwargs) -> Greenlet
+
         Create a new :class:`Greenlet` object and schedule it to run ``function(*args, **kwargs)``.
         This can be used as ``gevent.spawn`` or ``Greenlet.spawn``.
 
@@ -373,8 +542,10 @@ class Greenlet(greenlet):
     @classmethod
     def spawn_later(cls, seconds, *args, **kwargs):
         """
-        Create and return a new Greenlet object scheduled to run ``function(*args, **kwargs)``
-        in the future loop iteration *seconds* later. This can be used as ``Greenlet.spawn_later``
+        spawn_later(seconds, function, *args, **kwargs) -> Greenlet
+
+        Create and return a new `Greenlet` object scheduled to run ``function(*args, **kwargs)``
+        in a future loop iteration *seconds* later. This can be used as ``Greenlet.spawn_later``
         or ``gevent.spawn_later``.
 
         The arguments are passed to :meth:`Greenlet.__init__`.
@@ -433,9 +604,9 @@ class Greenlet(greenlet):
         self.__cancel_start()
 
         if self.dead:
-            self.__handle_death_before_start(exception)
+            self.__handle_death_before_start((exception,))
         else:
-            waiter = Waiter() if block else None
+            waiter = Waiter() if block else None # pylint:disable=undefined-variable
             self.parent.loop.run_callback(_kill, self, exception, waiter)
             if block:
                 waiter.get()
@@ -444,11 +615,17 @@ class Greenlet(greenlet):
         # thus it should not raise when the greenlet is already killed (= not started)
 
     def get(self, block=True, timeout=None):
-        """Return the result the greenlet has returned or re-raise the exception it has raised.
+        """
+        get(block=True, timeout=None) -> object
 
-        If block is ``False``, raise :class:`gevent.Timeout` if the greenlet is still alive.
-        If block is ``True``, unschedule the current greenlet until the result is available
-        or the timeout expires. In the latter case, :class:`gevent.Timeout` is raised.
+        Return the result the greenlet has returned or re-raise the
+        exception it has raised.
+
+        If block is ``False``, raise :class:`gevent.Timeout` if the
+        greenlet is still alive. If block is ``True``, unschedule the
+        current greenlet until the result is available or the timeout
+        expires. In the latter case, :class:`gevent.Timeout` is
+        raised.
         """
         if self.ready():
             if self.successful():
@@ -457,7 +634,7 @@ class Greenlet(greenlet):
         if not block:
             raise Timeout()
 
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             t = Timeout._start_new_or_dummy(timeout)
@@ -482,13 +659,16 @@ class Greenlet(greenlet):
             self._raise_exception()
 
     def join(self, timeout=None):
-        """Wait until the greenlet finishes or *timeout* expires.
-        Return ``None`` regardless.
+        """
+        join(timeout=None) -> None
+
+        Wait until the greenlet finishes or *timeout* expires. Return
+        ``None`` regardless.
         """
         if self.ready():
             return
 
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             t = Timeout._start_new_or_dummy(timeout)
@@ -509,7 +689,7 @@ class Greenlet(greenlet):
     def _report_result(self, result):
         self._exc_info = (None, None, None)
         self.value = result
-        if self._has_links() and not self._notifier:
+        if self._links and not self._notifier:
             self._notifier = self.parent.loop.run_callback(self._notify_links)
 
     def _report_error(self, exc_info):
@@ -519,7 +699,7 @@ class Greenlet(greenlet):
 
         self._exc_info = exc_info[0], exc_info[1], dump_traceback(exc_info[2])
 
-        if self._has_links() and not self._notifier:
+        if self._links and not self._notifier:
             self._notifier = self.parent.loop.run_callback(self._notify_links)
 
         try:
@@ -535,29 +715,38 @@ class Greenlet(greenlet):
             try:
                 result = self._run(*self.args, **self.kwargs)
             except: # pylint:disable=bare-except
-                self._report_error(sys.exc_info())
+                self._report_error(sys_exc_info())
                 return
             self._report_result(result)
         finally:
             self.__dict__.pop('_run', None)
-            self.__dict__.pop('args', None)
-            self.__dict__.pop('kwargs', None)
+            self.args = ()
+            self.kwargs.clear()
 
     def _run(self):
-        """Subclasses may override this method to take any number of arguments and keyword arguments.
+        """
+        Subclasses may override this method to take any number of
+        arguments and keyword arguments.
 
         .. versionadded:: 1.1a3
-            Previously, if no callable object was passed to the constructor, the spawned greenlet would
-            later fail with an AttributeError.
+            Previously, if no callable object was
+            passed to the constructor, the spawned greenlet would later
+            fail with an AttributeError.
         """
         # We usually override this in __init__
         # pylint: disable=method-hidden
         return
 
-    def rawlink(self, callback):
-        """Register a callable to be executed when the greenlet finishes execution.
+    def has_links(self):
+        return len(self._links)
 
-        The *callback* will be called with this instance as an argument.
+    def rawlink(self, callback):
+        """
+        Register a callable to be executed when the greenlet finishes
+        execution.
+
+        The *callback* will be called with this instance as an
+        argument.
 
         .. caution:: The callable will be called in the HUB greenlet.
         """
@@ -588,6 +777,14 @@ class Greenlet(greenlet):
         except ValueError:
             pass
 
+    def unlink_all(self):
+        """
+        Remove all the callbacks.
+
+        .. versionadded:: 1.3a2
+        """
+        del self._links[:]
+
     def link_value(self, callback, SpawnedLink=SuccessSpawnedLink):
         """
         Like :meth:`link` but *callback* is only notified when the greenlet
@@ -597,22 +794,34 @@ class Greenlet(greenlet):
         self.link(callback, SpawnedLink=SpawnedLink)
 
     def link_exception(self, callback, SpawnedLink=FailureSpawnedLink):
-        """Like :meth:`link` but *callback* is only notified when the greenlet dies because of an unhandled exception."""
+        """
+        Like :meth:`link` but *callback* is only notified when the
+        greenlet dies because of an unhandled exception.
+        """
         # pylint:disable=redefined-outer-name
         self.link(callback, SpawnedLink=SpawnedLink)
 
     def _notify_links(self):
         while self._links:
-            link = self._links.popleft() # pylint:disable=no-member
+            # Early links are allowed to remove later links
+            # before we get to them, and they're also allowed to
+            # add new links, so we have to be careful about iterating.
+
+            # We don't expect this list to be very large, so the time spent
+            # manipulating it should be small. a deque is probably not justified.
+            # Cython has optimizations to transform this into a memmove anyway.
+            link = self._links.pop(0)
             try:
                 link(self)
             except: # pylint:disable=bare-except
-                self.parent.handle_error((link, self), *sys.exc_info())
+                self.parent.handle_error((link, self), *sys_exc_info())
 
 
 class _dummy_event(object):
-    pending = False
-    active = False
+    __slots__ = ('pending', 'active')
+
+    def __init__(self):
+        self.pending = self.active = False
 
     def stop(self):
         pass
@@ -620,10 +829,11 @@ class _dummy_event(object):
     def start(self, cb): # pylint:disable=unused-argument
         raise AssertionError("Cannot start the dummy event")
 
+    def close(self):
+        pass
 
 _cancelled_start_event = _dummy_event()
 _start_completed_event = _dummy_event()
-del _dummy_event
 
 
 def _kill(glet, exception, waiter):
@@ -631,9 +841,9 @@ def _kill(glet, exception, waiter):
         glet.throw(exception)
     except: # pylint:disable=bare-except
         # XXX do we need this here?
-        glet.parent.handle_error(glet, *sys.exc_info())
+        glet.parent.handle_error(glet, *sys_exc_info())
     if waiter is not None:
-        waiter.switch()
+        waiter.switch(None)
 
 
 def joinall(greenlets, timeout=None, raise_error=False, count=None):
@@ -666,7 +876,7 @@ def _killall3(greenlets, exception, waiter):
             try:
                 g.throw(exception)
             except: # pylint:disable=bare-except
-                g.parent.handle_error(g, *sys.exc_info())
+                g.parent.handle_error(g, *sys_exc_info())
             if not g.dead:
                 diehards.append(g)
     waiter.switch(diehards)
@@ -678,7 +888,7 @@ def _killall(greenlets, exception):
             try:
                 g.throw(exception)
             except: # pylint:disable=bare-except
-                g.parent.handle_error(g, *sys.exc_info())
+                g.parent.handle_error(g, *sys_exc_info())
 
 
 def killall(greenlets, exception=GreenletExit, block=True, timeout=None):
@@ -713,7 +923,7 @@ def killall(greenlets, exception=GreenletExit, block=True, timeout=None):
         return
     loop = greenlets[0].loop
     if block:
-        waiter = Waiter()
+        waiter = Waiter() # pylint:disable=undefined-variable
         loop.run_callback(_killall3, greenlets, exception, waiter)
         t = Timeout._start_new_or_dummy(timeout)
         try:
@@ -725,20 +935,10 @@ def killall(greenlets, exception=GreenletExit, block=True, timeout=None):
     else:
         loop.run_callback(_killall, greenlets, exception)
 
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
 
-if PY3:
-    _meth_self = "__self__"
-else:
-    _meth_self = "im_self"
+_init()
 
-
-def getfuncname(func):
-    if not hasattr(func, _meth_self):
-        try:
-            funcname = func.__name__
-        except AttributeError:
-            pass
-        else:
-            if funcname != '<lambda>':
-                return funcname
-    return repr(func)
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent._greenlet')

python/gevent/libev/_corecffi_cdef.c

@@ -64,7 +64,8 @@ struct ev_loop {
 // Watcher types
 // base for all watchers
 struct ev_watcher{
-	GEVENT_STRUCT_DONE _;
+    void* data;
+    GEVENT_STRUCT_DONE _;
 };
 
 struct ev_io {
@@ -137,6 +138,9 @@ unsigned int ev_embeddable_backends (void);
 ev_tstamp ev_time (void);
 void ev_set_syserr_cb(void *);
 
+void ev_set_userdata(struct ev_loop*, void*);
+void* ev_userdata(struct ev_loop*);
+
 int ev_priority(void*);
 void ev_set_priority(void*, int);
 
@@ -212,10 +216,19 @@ void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
 void ev_sleep (ev_tstamp delay); /* sleep for a while */
 
 /* gevent callbacks */
-static int (*python_callback)(void* handle, int revents);
-static void (*python_handle_error)(void* handle, int revents);
-static void (*python_stop)(void* handle);
+/* These will be created as static functions at the end of the
+ * _source.c and must be declared there too.
+ */
+extern "Python" {
+	int python_callback(void* handle, int revents);
+	void python_handle_error(void* handle, int revents);
+	void python_stop(void* handle);
+	void python_check_callback(struct ev_loop*, void*, int);
+	void python_prepare_callback(struct ev_loop*, void*, int);
 
+	// libev specific
+	void _syserr_cb(char*);
+}
 /*
  * We use a single C callback for every watcher type, which in turn calls the
  * Python callbacks. The ev_watcher pointer type can be used for every watcher type
@@ -224,3 +237,7 @@ static void (*python_stop)(void* handle);
  * object.
  */
 static void _gevent_generic_callback(struct ev_loop* loop, struct ev_watcher* watcher, int revents);
+
+static void gevent_zero_check(struct ev_check* handle);
+static void gevent_zero_timer(struct ev_timer* handle);
+static void gevent_zero_prepare(struct ev_prepare* handle);

python/gevent/libev/_corecffi_source.c

@@ -13,9 +13,10 @@ static void
 _gevent_noop(struct ev_loop *_loop, struct ev_timer *w, int revents) { }
 
 void (*gevent_noop)(struct ev_loop *, struct ev_timer *, int) = &_gevent_noop;
-static int (*python_callback)(void* handle, int revents);
-static void (*python_handle_error)(void* handle, int revents);
-static void (*python_stop)(void* handle);
+
+static int python_callback(void* handle, int revents);
+static void python_handle_error(void* handle, int revents);
+static void python_stop(void* handle);
 
 static void _gevent_generic_callback(struct ev_loop* loop,
 									 struct ev_watcher* watcher,
@@ -30,7 +31,7 @@ static void _gevent_generic_callback(struct ev_loop* loop,
             // and allowing memory to be freed
             python_handle_error(handle, revents);
         break;
-        case 0:
+        case 1:
             // Code to stop the event. Note that if python_callback
             // has disposed of the last reference to the handle,
             // `watcher` could now be invalid/disposed memory!
@@ -38,8 +39,31 @@ static void _gevent_generic_callback(struct ev_loop* loop,
                 python_stop(handle);
             }
         break;
-        default:
-            assert(cb_result == 1);
+        case 2:
             // watcher is already stopped and dead, nothing to do.
+        break;
+        default:
+            fprintf(stderr,
+                    "WARNING: gevent: Unexpected return value %d from Python callback "
+                    "for watcher %p and handle %d\n",
+                    cb_result,
+                    watcher, handle);
+            // XXX: Possible leaking of resources here? Should we be
+            // closing the watcher?
     }
 }
+
+static void gevent_zero_timer(struct ev_timer* handle)
+{
+	memset(handle, 0, sizeof(struct ev_timer));
+}
+
+static void gevent_zero_check(struct ev_check* handle)
+{
+	memset(handle, 0, sizeof(struct ev_check));
+}
+
+static void gevent_zero_prepare(struct ev_prepare* handle)
+{
+	memset(handle, 0, sizeof(struct ev_prepare));
+}

python/gevent/libev/callbacks.c

@@ -1,42 +1,29 @@
 /* Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. */
+#include <stddef.h>
+#include "Python.h"
+#include "ev.h"
+#include "corecext.h"
+#include "callbacks.h"
 #ifdef Py_PYTHON_H
 
-/* the name changes depending on our file layout and --module-name option */
-#define _GEVENTLOOP struct __pyx_vtabstruct_6gevent_5libev_8corecext_loop
+#if PY_MAJOR_VERSION >= 3
+  #define PyInt_FromLong               PyLong_FromLong
+#endif
 
 
-static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* context) {
-    PyThreadState *tstate;
-    PyObject *type, *value, *traceback, *result;
-    tstate = PyThreadState_GET();
-    type = tstate->curexc_type;
-    if (!type)
-        return;
-    value = tstate->curexc_value;
-    traceback = tstate->curexc_traceback;
-    if (!value) value = Py_None;
-    if (!traceback) traceback = Py_None;
-
-    Py_INCREF(type);
-    Py_INCREF(value);
-    Py_INCREF(traceback);
-
-    PyErr_Clear();
-
-    result = ((_GEVENTLOOP *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
-
-    if (result) {
-        Py_DECREF(result);
-    }
-    else {
-        PyErr_Print();
-        PyErr_Clear();
-    }
-
-    Py_DECREF(type);
-    Py_DECREF(value);
-    Py_DECREF(traceback);
-}
+#ifndef CYTHON_INLINE
+  #if defined(__clang__)
+    #define CYTHON_INLINE __inline__ __attribute__ ((__unused__))
+  #elif defined(__GNUC__)
+    #define CYTHON_INLINE __inline__
+  #elif defined(_MSC_VER)
+    #define CYTHON_INLINE __inline
+  #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+    #define CYTHON_INLINE inline
+  #else
+    #define CYTHON_INLINE
+  #endif
+#endif
 
 
 static CYTHON_INLINE void gevent_check_signals(struct PyGeventLoopObject* loop) {
@@ -69,7 +56,7 @@ static void gevent_stop(PyObject* watcher, struct PyGeventLoopObject* loop) {
     error = 1;
     method = PyObject_GetAttrString(watcher, "stop");
     if (method) {
-        result = PyObject_Call(method, __pyx_empty_tuple, NULL);
+        result = PyObject_Call(method, _empty_tuple, NULL);
         if (result) {
             Py_DECREF(result);
             error = 0;
@@ -94,7 +81,7 @@ static void gevent_callback(struct PyGeventLoopObject* loop, PyObject* callback,
     Py_INCREF(watcher);
     gevent_check_signals(loop);
     if (args == Py_None) {
-        args = __pyx_empty_tuple;
+        args = _empty_tuple;
     }
     length = PyTuple_Size(args);
     if (length < 0) {
@@ -143,7 +130,7 @@ end:
 }
 
 
-static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb) {
+void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb) {
     /* no need for GIL here because it is only called from run_callbacks which already has GIL */
     PyObject *result, *callback, *args;
     if (!loop || !cb)
@@ -179,11 +166,16 @@ static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallback
     Py_DECREF(loop);
 }
 
+/*
+ * PyGeventWatcherObject is the first member of all the structs, so
+ * it is the same in all of them and they can all safely be cast to
+ * it. We could also use the *data member of the libev watcher objects.
+ */
 
 #undef DEFINE_CALLBACK
 #define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \
-    static void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) {                  \
-        struct PyGevent##WATCHER_TYPE##Object* watcher = GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher);    \
+    void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) {                  \
+        struct PyGeventWatcherObject* watcher = (struct PyGeventWatcherObject*)GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher);    \
         gevent_callback(watcher->loop, watcher->_callback, watcher->args, (PyObject*)watcher, c_watcher, revents); \
     }
 
@@ -191,7 +183,7 @@ static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallback
 DEFINE_CALLBACKS
 
 
-static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int revents) {
+void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int revents) {
     struct PyGeventLoopObject* loop;
     PyObject *result;
     GIL_DECLARE;
@@ -199,7 +191,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
     loop = GET_OBJECT(PyGeventLoopObject, watcher, _prepare);
     Py_INCREF(loop);
     gevent_check_signals(loop);
-    result = ((_GEVENTLOOP *)loop->__pyx_vtab)->_run_callbacks(loop);
+    result = gevent_loop_run_callbacks(loop);
     if (result) {
         Py_DECREF(result);
     }
@@ -211,15 +203,14 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
     GIL_RELEASE;
 }
 
-#if defined(_WIN32)
+/* This is only used on Win32 */
 
-static void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) {
+void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) {
     GIL_DECLARE;
     GIL_ENSURE;
     gevent_check_signals(GET_OBJECT(PyGeventLoopObject, watcher, _periodic_signal_checker));
     GIL_RELEASE;
 }
 
-#endif  /* _WIN32 */
 
 #endif  /* Py_PYTHON_H */

python/gevent/libev/callbacks.h

@@ -1,5 +1,9 @@
+struct ev_loop;
+struct PyGeventLoopObject;
+struct PyGeventCallbackObject;
+
 #define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \
-    static void gevent_callback_##WATCHER_LC(struct ev_loop *, void *, int);
+    void gevent_callback_##WATCHER_LC(struct ev_loop *, void *, int);
 
 
 #define DEFINE_CALLBACKS0              \
@@ -11,33 +15,24 @@
     DEFINE_CALLBACK(check, Check);     \
     DEFINE_CALLBACK(fork, Fork);       \
     DEFINE_CALLBACK(async, Async);     \
-    DEFINE_CALLBACK(stat, Stat);
+    DEFINE_CALLBACK(stat, Stat);       \
+    DEFINE_CALLBACK(child, Child);
 
 
-#ifndef _WIN32
-
-#define DEFINE_CALLBACKS               \
-    DEFINE_CALLBACKS0                  \
-    DEFINE_CALLBACK(child, Child)
-
-#else
-
 #define DEFINE_CALLBACKS DEFINE_CALLBACKS0
 
-#endif
-
 
 DEFINE_CALLBACKS
 
 
-static void gevent_run_callbacks(struct ev_loop *, void *, int);
-struct PyGeventLoopObject;
-static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* context);
-struct PyGeventCallbackObject;
-static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
+void gevent_run_callbacks(struct ev_loop *, void *, int);
 
-#if defined(_WIN32)
-static void gevent_periodic_signal_check(struct ev_loop *, void *, int);
-#endif
 
-static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) { }
+
+void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
+
+static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) {
+}
+
+/* Only used on Win32 */
+void gevent_periodic_signal_check(struct ev_loop *, void *, int);

python/gevent/libev/corecext.h

@@ -0,0 +1,147 @@
+/* Generated by Cython 0.28.5 */
+
+#ifndef __PYX_HAVE__gevent__libev__corecext
+#define __PYX_HAVE__gevent__libev__corecext
+
+struct PyGeventCallbackObject;
+struct PyGeventLoopObject;
+struct PyGeventWatcherObject;
+struct PyGeventIOObject;
+struct PyGeventTimerObject;
+struct PyGeventSignalObject;
+struct PyGeventIdleObject;
+struct PyGeventPrepareObject;
+struct PyGeventCheckObject;
+struct PyGeventForkObject;
+struct PyGeventAsyncObject;
+struct PyGeventChildObject;
+struct PyGeventStatObject;
+
+struct PyGeventCallbackObject {
+  PyObject_HEAD
+  PyObject *callback;
+  PyObject *args;
+  struct PyGeventCallbackObject *next;
+};
+
+struct PyGeventLoopObject {
+  PyObject_HEAD
+  struct __pyx_vtabstruct_6gevent_5libev_8corecext_loop *__pyx_vtab;
+  struct ev_prepare _prepare;
+  struct ev_timer _timer0;
+  struct ev_timer _periodic_signal_checker;
+  PyObject *error_handler;
+  struct ev_loop *_ptr;
+  struct __pyx_obj_6gevent_5libev_8corecext_CallbackFIFO *_callbacks;
+  int starting_timer_may_update_loop_time;
+  int _default;
+};
+
+struct PyGeventWatcherObject {
+  PyObject_HEAD
+  struct PyGeventLoopObject *loop;
+  PyObject *_callback;
+  PyObject *args;
+  struct ev_watcher *__pyx___watcher;
+  struct __pyx_t_6gevent_5libev_8corecext_start_and_stop *__pyx___ss;
+  unsigned int _flags;
+};
+
+struct PyGeventIOObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_io _watcher;
+};
+
+struct PyGeventTimerObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_timer _watcher;
+};
+
+struct PyGeventSignalObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_signal _watcher;
+};
+
+struct PyGeventIdleObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_idle _watcher;
+};
+
+struct PyGeventPrepareObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_prepare _watcher;
+};
+
+struct PyGeventCheckObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_check _watcher;
+};
+
+struct PyGeventForkObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_fork _watcher;
+};
+
+struct PyGeventAsyncObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_async _watcher;
+};
+
+struct PyGeventChildObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_child _watcher;
+};
+
+struct PyGeventStatObject {
+  struct PyGeventWatcherObject __pyx_base;
+  struct ev_stat _watcher;
+  PyObject *path;
+  PyObject *_paths;
+};
+
+#ifndef __PYX_HAVE_API__gevent__libev__corecext
+
+#ifndef __PYX_EXTERN_C
+  #ifdef __cplusplus
+    #define __PYX_EXTERN_C extern "C"
+  #else
+    #define __PYX_EXTERN_C extern
+  #endif
+#endif
+
+#ifndef DL_IMPORT
+  #define DL_IMPORT(_T) _T
+#endif
+
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventCallback_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventLoop_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventWatcher_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventIO_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventTimer_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventSignal_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventIdle_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventPrepare_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventCheck_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventFork_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventAsync_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventChild_Type;
+__PYX_EXTERN_C DL_IMPORT(PyTypeObject) PyGeventStat_Type;
+
+__PYX_EXTERN_C void gevent_handle_error(struct PyGeventLoopObject *, PyObject *);
+__PYX_EXTERN_C PyObject *gevent_loop_run_callbacks(struct PyGeventLoopObject *);
+
+__PYX_EXTERN_C PyObject *GEVENT_CORE_EVENTS;
+__PYX_EXTERN_C PyObject *_empty_tuple;
+
+#endif /* !__PYX_HAVE_API__gevent__libev__corecext */
+
+/* WARNING: the interface of the module init function changed in CPython 3.5. */
+/* It now returns a PyModuleDef instance instead of a PyModule instance. */
+
+#if PY_MAJOR_VERSION < 3
+PyMODINIT_FUNC initcorecext(void);
+#else
+PyMODINIT_FUNC PyInit_corecext(void);
+#endif
+
+#endif /* !__PYX_HAVE__gevent__libev__corecext */