Windows: Use 32-bit distribution of python

python/gevent/resolver/__init__.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+from _socket import gaierror
+from _socket import error
+from _socket import getservbyname
+from _socket import getaddrinfo
+
+from gevent._compat import string_types
+from gevent._compat import integer_types
+
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOCK_DGRAM
+from gevent.socket import SOL_TCP
+from gevent.socket import AI_CANONNAME
+from gevent.socket import EAI_SERVICE
+from gevent.socket import AF_INET
+from gevent.socket import AI_PASSIVE
+
+
+def _lookup_port(port, socktype):
+    # pylint:disable=too-many-branches
+    socktypes = []
+    if isinstance(port, string_types):
+        try:
+            port = int(port)
+        except ValueError:
+            try:
+                if socktype == 0:
+                    origport = port
+                    try:
+                        port = getservbyname(port, 'tcp')
+                        socktypes.append(SOCK_STREAM)
+                    except error:
+                        port = getservbyname(port, 'udp')
+                        socktypes.append(SOCK_DGRAM)
+                    else:
+                        try:
+                            if port == getservbyname(origport, 'udp'):
+                                socktypes.append(SOCK_DGRAM)
+                        except error:
+                            pass
+                elif socktype == SOCK_STREAM:
+                    port = getservbyname(port, 'tcp')
+                elif socktype == SOCK_DGRAM:
+                    port = getservbyname(port, 'udp')
+                else:
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+            except error as ex:
+                if 'not found' in str(ex):
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+                else:
+                    raise gaierror(str(ex))
+            except UnicodeEncodeError:
+                raise error('Int or String expected', port)
+    elif port is None:
+        port = 0
+    elif isinstance(port, integer_types):
+        pass
+    else:
+        raise error('Int or String expected', port, type(port))
+    port = int(port % 65536)
+    if not socktypes and socktype:
+        socktypes.append(socktype)
+    return port, socktypes
+
+hostname_types = tuple(set(string_types + (bytearray, bytes)))
+
+def _resolve_special(hostname, family):
+    if not isinstance(hostname, hostname_types):
+        raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
+
+    if hostname == '':
+        result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
+        if len(result) != 1:
+            raise error('wildcard resolved to multiple address')
+        return result[0][4][0]
+    return hostname
+
+
+class AbstractResolver(object):
+
+    def gethostbyname(self, hostname, family=AF_INET):
+        hostname = _resolve_special(hostname, family)
+        return self.gethostbyname_ex(hostname, family)[-1][0]
+
+    def gethostbyname_ex(self, hostname, family=AF_INET):
+        aliases = self._getaliases(hostname, family)
+        addresses = []
+        tuples = self.getaddrinfo(hostname, 0, family,
+                                  SOCK_STREAM,
+                                  SOL_TCP, AI_CANONNAME)
+        canonical = tuples[0][3]
+        for item in tuples:
+            addresses.append(item[4][0])
+        # XXX we just ignore aliases
+        return (canonical, aliases, addresses)
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        raise NotImplementedError()
+
+    def _getaliases(self, hostname, family):
+        # pylint:disable=unused-argument
+        return []

python/gevent/resolver/ares.py

@@ -0,0 +1,357 @@
+# Copyright (c) 2011-2015 Denis Bilenko. See LICENSE for details.
+"""
+c-ares based hostname resolver.
+"""
+from __future__ import absolute_import, print_function, division
+import os
+import sys
+
+from _socket import getaddrinfo
+from _socket import gaierror
+from _socket import error
+
+from gevent._compat import string_types
+from gevent._compat import text_type
+
+from gevent._compat import reraise
+from gevent._compat import PY3
+
+from gevent.hub import Waiter
+from gevent.hub import get_hub
+
+from gevent.socket import AF_UNSPEC
+from gevent.socket import AF_INET
+from gevent.socket import AF_INET6
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOCK_DGRAM
+from gevent.socket import SOCK_RAW
+from gevent.socket import AI_NUMERICHOST
+
+from gevent._config import config
+from gevent._config import AresSettingMixin
+
+from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
+from . import _lookup_port as lookup_port
+from . import _resolve_special
+from . import AbstractResolver
+
+__all__ = ['Resolver']
+
+
+class Resolver(AbstractResolver):
+    """
+    Implementation of the resolver API using the `c-ares`_ library.
+
+    This implementation uses the c-ares library to handle name
+    resolution. c-ares is natively asynchronous at the socket level
+    and so integrates well into gevent's event loop.
+
+    In comparison to :class:`gevent.resolver_thread.Resolver` (which
+    delegates to the native system resolver), the implementation is
+    much more complex. In addition, there have been reports of it not
+    properly honoring certain system configurations (for example, the
+    order in which IPv4 and IPv6 results are returned may not match
+    the threaded resolver). However, because it does not use threads,
+    it may scale better for applications that make many lookups.
+
+    There are some known differences from the system resolver:
+
+    - ``gethostbyname_ex`` and ``gethostbyaddr`` may return different
+      for the ``aliaslist`` tuple member. (Sometimes the same,
+      sometimes in a different order, sometimes a different alias
+      altogether.)
+    - ``gethostbyname_ex`` may return the ``ipaddrlist`` in a different order.
+    - ``getaddrinfo`` does not return ``SOCK_RAW`` results.
+    - ``getaddrinfo`` may return results in a different order.
+    - Handling of ``.local`` (mDNS) names may be different, even if they are listed in
+      the hosts file.
+    - c-ares will not resolve ``broadcasthost``, even if listed in the hosts file.
+    - This implementation may raise ``gaierror(4)`` where the system implementation would raise
+      ``herror(1)``.
+    - The results for ``localhost`` may be different. In particular, some system
+      resolvers will return more results from ``getaddrinfo`` than c-ares does,
+      such as SOCK_DGRAM results, and c-ares may report more ips on a multi-homed
+      host.
+
+    .. caution:: This module is considered extremely experimental on PyPy, and
+       due to its implementation in cython, it may be slower. It may also lead to
+       interpreter crashes.
+
+    .. _c-ares: http://c-ares.haxx.se
+    """
+
+    ares_class = channel
+
+    def __init__(self, hub=None, use_environ=True, **kwargs):
+        if hub is None:
+            hub = get_hub()
+        self.hub = hub
+        if use_environ:
+            for setting in config.settings.values():
+                if isinstance(setting, AresSettingMixin):
+                    value = setting.get()
+                    if value is not None:
+                        kwargs.setdefault(setting.kwarg_name, value)
+        self.ares = self.ares_class(hub.loop, **kwargs)
+        self.pid = os.getpid()
+        self.params = kwargs
+        self.fork_watcher = hub.loop.fork(ref=False)
+        self.fork_watcher.start(self._on_fork)
+
+    def __repr__(self):
+        return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.ares)
+
+    def _on_fork(self):
+        # NOTE: See comment in gevent.hub.reinit.
+        pid = os.getpid()
+        if pid != self.pid:
+            self.hub.loop.run_callback(self.ares.destroy)
+            self.ares = self.ares_class(self.hub.loop, **self.params)
+            self.pid = pid
+
+    def close(self):
+        if self.ares is not None:
+            self.hub.loop.run_callback(self.ares.destroy)
+            self.ares = None
+        self.fork_watcher.stop()
+
+    def gethostbyname(self, hostname, family=AF_INET):
+        hostname = _resolve_special(hostname, family)
+        return self.gethostbyname_ex(hostname, family)[-1][0]
+
+    def gethostbyname_ex(self, hostname, family=AF_INET):
+        if PY3:
+            if isinstance(hostname, str):
+                hostname = hostname.encode('idna')
+            elif not isinstance(hostname, (bytes, bytearray)):
+                raise TypeError('Expected es(idna), not %s' % type(hostname).__name__)
+        else:
+            if isinstance(hostname, text_type):
+                hostname = hostname.encode('ascii')
+            elif not isinstance(hostname, str):
+                raise TypeError('Expected string, not %s' % type(hostname).__name__)
+
+        while True:
+            ares = self.ares
+            try:
+                waiter = Waiter(self.hub)
+                ares.gethostbyname(waiter, hostname, family)
+                result = waiter.get()
+                if not result[-1]:
+                    raise gaierror(-5, 'No address associated with hostname')
+                return result
+            except gaierror:
+                if ares is self.ares:
+                    if hostname == b'255.255.255.255':
+                        # The stdlib handles this case in 2.7 and 3.x, but ares does not.
+                        # It is tested by test_socket.py in 3.4.
+                        # HACK: So hardcode the expected return.
+                        return ('255.255.255.255', [], ['255.255.255.255'])
+                    raise
+                # "self.ares is not ares" means channel was destroyed (because we were forked)
+
+    def _lookup_port(self, port, socktype):
+        return lookup_port(port, socktype)
+
+    def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        # pylint:disable=too-many-locals,too-many-branches
+        if isinstance(host, text_type):
+            host = host.encode('idna')
+        elif not isinstance(host, str) or (flags & AI_NUMERICHOST):
+            # this handles cases which do not require network access
+            # 1) host is None
+            # 2) host is of an invalid type
+            # 3) AI_NUMERICHOST flag is set
+            return getaddrinfo(host, port, family, socktype, proto, flags)
+            # we also call _socket.getaddrinfo below if family is not one of AF_*
+
+        port, socktypes = self._lookup_port(port, socktype)
+
+        socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
+        if socktypes:
+            socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
+        if proto:
+            socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
+
+        ares = self.ares
+
+        if family == AF_UNSPEC:
+            ares_values = Values(self.hub, 2)
+            ares.gethostbyname(ares_values, host, AF_INET)
+            ares.gethostbyname(ares_values, host, AF_INET6)
+        elif family == AF_INET:
+            ares_values = Values(self.hub, 1)
+            ares.gethostbyname(ares_values, host, AF_INET)
+        elif family == AF_INET6:
+            ares_values = Values(self.hub, 1)
+            ares.gethostbyname(ares_values, host, AF_INET6)
+        else:
+            raise gaierror(5, 'ai_family not supported: %r' % (family, ))
+
+        values = ares_values.get()
+        if len(values) == 2 and values[0] == values[1]:
+            values.pop()
+
+        result = []
+        result4 = []
+        result6 = []
+
+        for addrs in values:
+            if addrs.family == AF_INET:
+                for addr in addrs[-1]:
+                    sockaddr = (addr, port)
+                    for socktype4, proto4 in socktype_proto:
+                        result4.append((AF_INET, socktype4, proto4, '', sockaddr))
+            elif addrs.family == AF_INET6:
+                for addr in addrs[-1]:
+                    if addr == '::1':
+                        dest = result
+                    else:
+                        dest = result6
+                    sockaddr = (addr, port, 0, 0)
+                    for socktype6, proto6 in socktype_proto:
+                        dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
+
+        # As of 2016, some platforms return IPV6 first and some do IPV4 first,
+        # and some might even allow configuration of which is which. For backwards
+        # compatibility with earlier releases (but not necessarily resolver_thread!)
+        # we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
+        result += result4 + result6
+
+        if not result:
+            raise gaierror(-5, 'No address associated with hostname')
+
+        return result
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        while True:
+            ares = self.ares
+            try:
+                return self._getaddrinfo(host, port, family, socktype, proto, flags)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+    def _gethostbyaddr(self, ip_address):
+        if PY3:
+            if isinstance(ip_address, str):
+                ip_address = ip_address.encode('idna')
+            elif not isinstance(ip_address, (bytes, bytearray)):
+                raise TypeError('Expected es(idna), not %s' % type(ip_address).__name__)
+        else:
+            if isinstance(ip_address, text_type):
+                ip_address = ip_address.encode('ascii')
+            elif not isinstance(ip_address, str):
+                raise TypeError('Expected string, not %s' % type(ip_address).__name__)
+
+        waiter = Waiter(self.hub)
+        try:
+            self.ares.gethostbyaddr(waiter, ip_address)
+            return waiter.get()
+        except InvalidIP:
+            result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
+            if not result:
+                raise
+            _ip_address = result[0][-1][0]
+            if isinstance(_ip_address, text_type):
+                _ip_address = _ip_address.encode('ascii')
+            if _ip_address == ip_address:
+                raise
+            waiter.clear()
+            self.ares.gethostbyaddr(waiter, _ip_address)
+            return waiter.get()
+
+    def gethostbyaddr(self, ip_address):
+        ip_address = _resolve_special(ip_address, AF_UNSPEC)
+        while True:
+            ares = self.ares
+            try:
+                return self._gethostbyaddr(ip_address)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+    def _getnameinfo(self, sockaddr, flags):
+        if not isinstance(flags, int):
+            raise TypeError('an integer is required')
+        if not isinstance(sockaddr, tuple):
+            raise TypeError('getnameinfo() argument 1 must be a tuple')
+
+        address = sockaddr[0]
+        if not PY3 and isinstance(address, text_type):
+            address = address.encode('ascii')
+
+        if not isinstance(address, string_types):
+            raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
+
+        port = sockaddr[1]
+        if not isinstance(port, int):
+            raise TypeError('port must be an integer, not %s' % type(port))
+
+        waiter = Waiter(self.hub)
+        result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
+        if not result:
+            reraise(*sys.exc_info())
+        elif len(result) != 1:
+            raise error('sockaddr resolved to multiple addresses')
+        family, _socktype, _proto, _name, address = result[0]
+
+        if family == AF_INET:
+            if len(sockaddr) != 2:
+                raise error("IPv4 sockaddr must be 2 tuple")
+        elif family == AF_INET6:
+            address = address[:2] + sockaddr[2:]
+
+        self.ares.getnameinfo(waiter, address, flags)
+        node, service = waiter.get()
+
+        if service is None:
+            if PY3:
+                # ares docs: "If the query did not complete
+                # successfully, or one of the values was not
+                # requested, node or service will be NULL ". Python 2
+                # allows that for the service, but Python 3 raises
+                # an error. This is tested by test_socket in py 3.4
+                err = gaierror('nodename nor servname provided, or not known')
+                err.errno = 8
+                raise err
+            service = '0'
+        return node, service
+
+    def getnameinfo(self, sockaddr, flags):
+        while True:
+            ares = self.ares
+            try:
+                return self._getnameinfo(sockaddr, flags)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+
+class Values(object):
+    # helper to collect multiple values; ignore errors unless nothing has succeeded
+    # QQQ could probably be moved somewhere - hub.py?
+
+    __slots__ = ['count', 'values', 'error', 'waiter']
+
+    def __init__(self, hub, count):
+        self.count = count
+        self.values = []
+        self.error = None
+        self.waiter = Waiter(hub)
+
+    def __call__(self, source):
+        self.count -= 1
+        if source.exception is None:
+            self.values.append(source.value)
+        else:
+            self.error = source.exception
+        if self.count <= 0:
+            self.waiter.switch(None)
+
+    def get(self):
+        self.waiter.get()
+        if self.values:
+            return self.values
+        assert error is not None
+        raise self.error # pylint:disable=raising-bad-type

python/gevent/resolver/blocking.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+import _socket
+
+class Resolver(object):
+    """
+    A resolver that directly uses the system's resolver functions.
+
+    .. caution::
+
+        This resolver is *not* cooperative.
+
+    This resolver has the lowest overhead of any resolver and
+    typically approaches the speed of the unmodified :mod:`socket`
+    functions. However, it is not cooperative, so if name resolution
+    blocks, the entire thread and all its greenlets will be blocked.
+
+    This can be useful during debugging, or it may be a good choice if
+    your operating system provides a good caching resolver (such as
+    macOS's Directory Services) that is usually very fast and
+    functionally non-blocking.
+
+    .. versionchanged:: 1.3a2
+       This was previously undocumented and existed in :mod:`gevent.socket`.
+
+    """
+
+    def __init__(self, hub=None):
+        pass
+
+    def close(self):
+        pass
+
+    for method in (
+            'gethostbyname',
+            'gethostbyname_ex',
+            'getaddrinfo',
+            'gethostbyaddr',
+            'getnameinfo'
+    ):
+        locals()[method] = staticmethod(getattr(_socket, method))

python/gevent/resolver/cares.pyx

@@ -0,0 +1,464 @@
+# Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details.
+# Automatic pickling of cdef classes was added in 0.26. Unfortunately it
+# seems to be buggy (at least for the `result` class) and produces code that
+# can't compile ("local variable 'result' referenced before assignment").
+# See https://github.com/cython/cython/issues/1786
+# cython: auto_pickle=False
+cimport libcares as cares
+import sys
+
+from cpython.ref cimport Py_INCREF
+from cpython.ref cimport Py_DECREF
+from cpython.mem cimport PyMem_Malloc
+from cpython.mem cimport PyMem_Free
+
+from _socket import gaierror
+
+
+__all__ = ['channel']
+
+cdef object string_types
+cdef object text_type
+
+if sys.version_info[0] >= 3:
+    string_types = str,
+    text_type = str
+else:
+    string_types = __builtins__.basestring,
+    text_type = __builtins__.unicode
+
+TIMEOUT = 1
+
+DEF EV_READ = 1
+DEF EV_WRITE = 2
+
+
+cdef extern from "dnshelper.c":
+    int AF_INET
+    int AF_INET6
+
+    struct hostent:
+        char* h_name
+        int h_addrtype
+
+    struct sockaddr_t "sockaddr":
+        pass
+
+    struct ares_channeldata:
+        pass
+
+    object parse_h_name(hostent*)
+    object parse_h_aliases(hostent*)
+    object parse_h_addr_list(hostent*)
+    void* create_object_from_hostent(void*)
+
+    # this imports _socket lazily
+    object PyUnicode_FromString(char*)
+    int PyTuple_Check(object)
+    int PyArg_ParseTuple(object, char*, ...) except 0
+    struct sockaddr_in6:
+        pass
+    int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6)
+
+
+    void memset(void*, int, int)
+
+
+ARES_SUCCESS = cares.ARES_SUCCESS
+ARES_ENODATA = cares.ARES_ENODATA
+ARES_EFORMERR = cares.ARES_EFORMERR
+ARES_ESERVFAIL = cares.ARES_ESERVFAIL
+ARES_ENOTFOUND = cares.ARES_ENOTFOUND
+ARES_ENOTIMP = cares.ARES_ENOTIMP
+ARES_EREFUSED = cares.ARES_EREFUSED
+ARES_EBADQUERY = cares.ARES_EBADQUERY
+ARES_EBADNAME = cares.ARES_EBADNAME
+ARES_EBADFAMILY = cares.ARES_EBADFAMILY
+ARES_EBADRESP = cares.ARES_EBADRESP
+ARES_ECONNREFUSED = cares.ARES_ECONNREFUSED
+ARES_ETIMEOUT = cares.ARES_ETIMEOUT
+ARES_EOF = cares.ARES_EOF
+ARES_EFILE = cares.ARES_EFILE
+ARES_ENOMEM = cares.ARES_ENOMEM
+ARES_EDESTRUCTION = cares.ARES_EDESTRUCTION
+ARES_EBADSTR = cares.ARES_EBADSTR
+ARES_EBADFLAGS = cares.ARES_EBADFLAGS
+ARES_ENONAME = cares.ARES_ENONAME
+ARES_EBADHINTS = cares.ARES_EBADHINTS
+ARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED
+ARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI
+ARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS
+ARES_ECANCELLED = cares.ARES_ECANCELLED
+
+ARES_FLAG_USEVC = cares.ARES_FLAG_USEVC
+ARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY
+ARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC
+ARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE
+ARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN
+ARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH
+ARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES
+ARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP
+
+
+_ares_errors = dict([
+                (cares.ARES_SUCCESS, 'ARES_SUCCESS'),
+                (cares.ARES_ENODATA, 'ARES_ENODATA'),
+                (cares.ARES_EFORMERR, 'ARES_EFORMERR'),
+                (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
+                (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
+                (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
+                (cares.ARES_EREFUSED, 'ARES_EREFUSED'),
+                (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
+                (cares.ARES_EBADNAME, 'ARES_EBADNAME'),
+                (cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),
+                (cares.ARES_EBADRESP, 'ARES_EBADRESP'),
+                (cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),
+                (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
+                (cares.ARES_EOF, 'ARES_EOF'),
+                (cares.ARES_EFILE, 'ARES_EFILE'),
+                (cares.ARES_ENOMEM, 'ARES_ENOMEM'),
+                (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
+                (cares.ARES_EBADSTR, 'ARES_EBADSTR'),
+                (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
+                (cares.ARES_ENONAME, 'ARES_ENONAME'),
+                (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
+                (cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),
+                (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
+                (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
+                (cares.ARES_ECANCELLED, 'ARES_ECANCELLED')])
+
+
+# maps c-ares flag to _socket module flag
+_cares_flag_map = None
+
+
+cdef _prepare_cares_flag_map():
+    global _cares_flag_map
+    import _socket
+    _cares_flag_map = [
+        (getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST),
+        (getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV),
+        (getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN),
+        (getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD),
+        (getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)]
+
+
+cpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE):
+    if _cares_flag_map is None:
+        _prepare_cares_flag_map()
+    for socket_flag, cares_flag in _cares_flag_map:
+        if socket_flag & flags:
+            default |= cares_flag
+            flags &= ~socket_flag
+        if not flags:
+            return default
+    raise gaierror(-1, "Bad value for ai_flags: 0x%x" % flags)
+
+
+cpdef strerror(code):
+    return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code))
+
+
+class InvalidIP(ValueError):
+    pass
+
+
+cdef void gevent_sock_state_callback(void *data, int s, int read, int write):
+    if not data:
+        return
+    cdef channel ch = <channel>data
+    ch._sock_state_callback(s, read, write)
+
+
+cdef class result:
+    cdef public object value
+    cdef public object exception
+
+    def __init__(self, object value=None, object exception=None):
+        self.value = value
+        self.exception = exception
+
+    def __repr__(self):
+        if self.exception is None:
+            return '%s(%r)' % (self.__class__.__name__, self.value)
+        elif self.value is None:
+            return '%s(exception=%r)' % (self.__class__.__name__, self.exception)
+        else:
+            return '%s(value=%r, exception=%r)' % (self.__class__.__name__, self.value, self.exception)
+        # add repr_recursive precaution
+
+    def successful(self):
+        return self.exception is None
+
+    def get(self):
+        if self.exception is not None:
+            raise self.exception
+        return self.value
+
+
+class ares_host_result(tuple):
+
+    def __new__(cls, family, iterable):
+        cdef object self = tuple.__new__(cls, iterable)
+        self.family = family
+        return self
+
+    def __getnewargs__(self):
+        return (self.family, tuple(self))
+
+
+cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host):
+    cdef channel channel
+    cdef object callback
+    channel, callback = <tuple>arg
+    Py_DECREF(<tuple>arg)
+    cdef object host_result
+    try:
+        if status or not host:
+            callback(result(None, gaierror(status, strerror(status))))
+        else:
+            try:
+                host_result = ares_host_result(host.h_addrtype, (parse_h_name(host), parse_h_aliases(host), parse_h_addr_list(host)))
+            except:
+                callback(result(None, sys.exc_info()[1]))
+            else:
+                callback(result(host_result))
+    except:
+        channel.loop.handle_error(callback, *sys.exc_info())
+
+
+cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service):
+    cdef channel channel
+    cdef object callback
+    channel, callback = <tuple>arg
+    Py_DECREF(<tuple>arg)
+    cdef object node
+    cdef object service
+    try:
+        if status:
+            callback(result(None, gaierror(status, strerror(status))))
+        else:
+            if c_node:
+                node = PyUnicode_FromString(c_node)
+            else:
+                node = None
+            if c_service:
+                service = PyUnicode_FromString(c_service)
+            else:
+                service = None
+            callback(result((node, service)))
+    except:
+        channel.loop.handle_error(callback, *sys.exc_info())
+
+
+cdef class channel:
+
+    cdef public object loop
+    cdef ares_channeldata* channel
+    cdef public dict _watchers
+    cdef public object _timer
+
+    def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None,
+                 udp_port=None, tcp_port=None, servers=None):
+        cdef ares_channeldata* channel = NULL
+        cdef cares.ares_options options
+        memset(&options, 0, sizeof(cares.ares_options))
+        cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB
+        options.sock_state_cb = <void*>gevent_sock_state_callback
+        options.sock_state_cb_data = <void*>self
+        if flags is not None:
+            options.flags = int(flags)
+            optmask |= cares.ARES_OPT_FLAGS
+        if timeout is not None:
+            options.timeout = int(float(timeout) * 1000)
+            optmask |= cares.ARES_OPT_TIMEOUTMS
+        if tries is not None:
+            options.tries = int(tries)
+            optmask |= cares.ARES_OPT_TRIES
+        if ndots is not None:
+            options.ndots = int(ndots)
+            optmask |= cares.ARES_OPT_NDOTS
+        if udp_port is not None:
+            options.udp_port = int(udp_port)
+            optmask |= cares.ARES_OPT_UDP_PORT
+        if tcp_port is not None:
+            options.tcp_port = int(tcp_port)
+            optmask |= cares.ARES_OPT_TCP_PORT
+        cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL)  # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK?
+        if result:
+            raise gaierror(result, strerror(result))
+        result = cares.ares_init_options(&channel, &options, optmask)
+        if result:
+            raise gaierror(result, strerror(result))
+        self._timer = loop.timer(TIMEOUT, TIMEOUT)
+        self._watchers = {}
+        self.channel = channel
+        try:
+            if servers is not None:
+                self.set_servers(servers)
+            self.loop = loop
+        except:
+            self.destroy()
+            raise
+
+    def __repr__(self):
+        args = (self.__class__.__name__, id(self), self._timer, len(self._watchers))
+        return '<%s at 0x%x _timer=%r _watchers[%s]>' % args
+
+    def destroy(self):
+        if self.channel:
+            # XXX ares_library_cleanup?
+            cares.ares_destroy(self.channel)
+            self.channel = NULL
+            self._watchers.clear()
+            self._timer.stop()
+            self.loop = None
+
+    def __dealloc__(self):
+        if self.channel:
+            # XXX ares_library_cleanup?
+            cares.ares_destroy(self.channel)
+            self.channel = NULL
+
+    cpdef set_servers(self, servers=None):
+        if not self.channel:
+            raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
+        if not servers:
+            servers = []
+        if isinstance(servers, string_types):
+            servers = servers.split(',')
+        cdef int length = len(servers)
+        cdef int result, index
+        cdef char* string
+        cdef cares.ares_addr_node* c_servers
+        if length <= 0:
+            result = cares.ares_set_servers(self.channel, NULL)
+        else:
+            c_servers = <cares.ares_addr_node*>PyMem_Malloc(sizeof(cares.ares_addr_node) * length)
+            if not c_servers:
+                raise MemoryError
+            try:
+                index = 0
+                for server in servers:
+                    if isinstance(server, unicode):
+                        server = server.encode('ascii')
+                    string = <char*?>server
+                    if cares.ares_inet_pton(AF_INET, string, &c_servers[index].addr) > 0:
+                        c_servers[index].family = AF_INET
+                    elif cares.ares_inet_pton(AF_INET6, string, &c_servers[index].addr) > 0:
+                        c_servers[index].family = AF_INET6
+                    else:
+                        raise InvalidIP(repr(string))
+                    c_servers[index].next = &c_servers[index] + 1
+                    index += 1
+                    if index >= length:
+                        break
+                c_servers[length - 1].next = NULL
+                index = cares.ares_set_servers(self.channel, c_servers)
+                if index:
+                    raise ValueError(strerror(index))
+            finally:
+                PyMem_Free(c_servers)
+
+    # this crashes c-ares
+    #def cancel(self):
+    #    cares.ares_cancel(self.channel)
+
+    cdef _sock_state_callback(self, int socket, int read, int write):
+        if not self.channel:
+            return
+        cdef object watcher = self._watchers.get(socket)
+        cdef int events = 0
+        if read:
+            events |= EV_READ
+        if write:
+            events |= EV_WRITE
+        if watcher is None:
+            if not events:
+                return
+            watcher = self.loop.io(socket, events)
+            self._watchers[socket] = watcher
+        elif events:
+            if watcher.events == events:
+                return
+            watcher.stop()
+            watcher.events = events
+        else:
+            watcher.stop()
+            watcher.close()
+            self._watchers.pop(socket, None)
+            if not self._watchers:
+                self._timer.stop()
+            return
+        watcher.start(self._process_fd, watcher, pass_events=True)
+        self._timer.again(self._on_timer)
+
+    def _on_timer(self):
+        cares.ares_process_fd(self.channel, cares.ARES_SOCKET_BAD, cares.ARES_SOCKET_BAD)
+
+    def _process_fd(self, int events, object watcher):
+        if not self.channel:
+            return
+        cdef int read_fd = watcher.fd
+        cdef int write_fd = read_fd
+        if not (events & EV_READ):
+            read_fd = cares.ARES_SOCKET_BAD
+        if not (events & EV_WRITE):
+            write_fd = cares.ARES_SOCKET_BAD
+        cares.ares_process_fd(self.channel, read_fd, write_fd)
+
+    def gethostbyname(self, object callback, char* name, int family=AF_INET):
+        if not self.channel:
+            raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
+        # note that for file lookups still AF_INET can be returned for AF_INET6 request
+        cdef object arg = (self, callback)
+        Py_INCREF(arg)
+        cares.ares_gethostbyname(self.channel, name, family, <void*>gevent_ares_host_callback, <void*>arg)
+
+    def gethostbyaddr(self, object callback, char* addr):
+        if not self.channel:
+            raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
+        # will guess the family
+        cdef char addr_packed[16]
+        cdef int family
+        cdef int length
+        if cares.ares_inet_pton(AF_INET, addr, addr_packed) > 0:
+            family = AF_INET
+            length = 4
+        elif cares.ares_inet_pton(AF_INET6, addr, addr_packed) > 0:
+            family = AF_INET6
+            length = 16
+        else:
+            raise InvalidIP(repr(addr))
+        cdef object arg = (self, callback)
+        Py_INCREF(arg)
+        cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, <void*>gevent_ares_host_callback, <void*>arg)
+
+    cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags):
+        if not self.channel:
+            raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
+        cdef char* hostp = NULL
+        cdef int port = 0
+        cdef int flowinfo = 0
+        cdef int scope_id = 0
+        cdef sockaddr_in6 sa6
+        if not PyTuple_Check(sockaddr):
+            raise TypeError('expected a tuple, got %r' % (sockaddr, ))
+        PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id)
+        if port < 0 or port > 65535:
+            raise gaierror(-8, 'Invalid value for port: %r' % port)
+        cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
+        if length <= 0:
+            raise InvalidIP(repr(hostp))
+        cdef object arg = (self, callback)
+        Py_INCREF(arg)
+        cdef sockaddr_t* x = <sockaddr_t*>&sa6
+        cares.ares_getnameinfo(self.channel, x, length, flags, <void*>gevent_ares_nameinfo_callback, <void*>arg)
+
+    def getnameinfo(self, object callback, tuple sockaddr, int flags):
+        try:
+            flags = _convert_cares_flags(flags)
+        except gaierror:
+            # The stdlib just ignores bad flags
+            flags = 0
+        return self._getnameinfo(callback, sockaddr, flags)

python/gevent/resolver/cares_ntop.h

@@ -0,0 +1,7 @@
+#ifdef CARES_EMBED
+#include "ares_setup.h"
+#include "ares.h"
+#else
+#include <arpa/inet.h>
+#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z)
+#endif

python/gevent/resolver/cares_pton.h

@@ -0,0 +1,8 @@
+#ifdef CARES_EMBED
+#include "ares_setup.h"
+#include "ares_inet_net_pton.h"
+#else
+#include <arpa/inet.h>
+#define ares_inet_pton(x,y,z) inet_pton(x,y,z)
+#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z)
+#endif

python/gevent/resolver/dnshelper.c

@@ -0,0 +1,159 @@
+/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */
+#include "Python.h"
+#ifdef CARES_EMBED
+#include "ares_setup.h"
+#endif
+
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+
+#include "ares.h"
+
+#include "cares_ntop.h"
+#include "cares_pton.h"
+
+#if PY_VERSION_HEX < 0x02060000
+  #define PyUnicode_FromString         PyString_FromString
+#elif PY_MAJOR_VERSION < 3
+  #define PyUnicode_FromString         PyBytes_FromString
+#endif
+
+
+static PyObject* _socket_error = 0;
+
+static PyObject*
+get_socket_object(PyObject** pobject, const char* name)
+{
+    if (!*pobject) {
+        PyObject* _socket;
+        _socket = PyImport_ImportModule("_socket");
+        if (_socket) {
+            *pobject = PyObject_GetAttrString(_socket, name);
+            if (!*pobject) {
+                PyErr_WriteUnraisable(Py_None);
+            }
+            Py_DECREF(_socket);
+        }
+        else {
+            PyErr_WriteUnraisable(Py_None);
+        }
+        if (!*pobject) {
+            *pobject = PyExc_IOError;
+        }
+    }
+    return *pobject;
+}
+
+
+static int
+gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) {
+    int status = -1;
+    PyObject* tmp;
+    if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) {
+        tmp = PyUnicode_FromString(tmpbuf);
+        if (tmp) {
+            status = PyList_Append(list, tmp);
+            Py_DECREF(tmp);
+        }
+    }
+    return status;
+}
+
+
+static PyObject*
+parse_h_name(struct hostent *h)
+{
+    return PyUnicode_FromString(h->h_name);
+}
+
+
+static PyObject*
+parse_h_aliases(struct hostent *h)
+{
+    char **pch;
+    PyObject *result = NULL;
+    PyObject *tmp;
+
+    result = PyList_New(0);
+
+    if (result && h->h_aliases) {
+        for (pch = h->h_aliases; *pch != NULL; pch++) {
+            if (*pch != h->h_name && strcmp(*pch, h->h_name)) {
+                int status;
+                tmp = PyUnicode_FromString(*pch);
+                if (tmp == NULL) {
+                    break;
+                }
+
+                status = PyList_Append(result, tmp);
+                Py_DECREF(tmp);
+
+                if (status) {
+                    break;
+                }
+            }
+        }
+    }
+
+    return result;
+}
+
+
+static PyObject *
+parse_h_addr_list(struct hostent *h)
+{
+    char **pch;
+    PyObject *result = NULL;
+
+    result = PyList_New(0);
+
+    if (result) {
+        switch (h->h_addrtype) {
+            case AF_INET:
+            {
+                char tmpbuf[sizeof "255.255.255.255"];
+                for (pch = h->h_addr_list; *pch != NULL; pch++) {
+                    if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) {
+                        break;
+                    }
+                }
+                break;
+            }
+            case AF_INET6:
+            {
+                char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")];
+                for (pch = h->h_addr_list; *pch != NULL; pch++) {
+                    if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) {
+                        break;
+                    }
+                }
+                break;
+            }
+            default:
+                PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family");
+                Py_DECREF(result);
+                result = NULL;
+        }
+    }
+
+    return result;
+}
+
+
+static int
+gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) {
+    if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) {
+        ((struct sockaddr_in*)sa6)->sin_family = AF_INET;
+        ((struct sockaddr_in*)sa6)->sin_port = htons(port);
+        return sizeof(struct sockaddr_in);
+    }
+    else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) {
+        sa6->sin6_family = AF_INET6;
+        sa6->sin6_port = htons(port);
+        sa6->sin6_flowinfo = flowinfo;
+        sa6->sin6_scope_id = scope_id;
+        return sizeof(struct sockaddr_in6);
+    }
+    return -1;
+}

python/gevent/resolver/dnspython.py

@@ -0,0 +1,662 @@
+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+# Portions of this code taken from the gogreen project:
+#   http://github.com/slideinc/gogreen
+#
+# Copyright (c) 2005-2010 Slide, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of the author nor the names of other
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Portions of this code taken from the eventlet project:
+# https://github.com/eventlet/eventlet/blob/master/eventlet/support/greendns.py
+
+# Unless otherwise noted, the files in Eventlet are under the following MIT license:
+
+# Copyright (c) 2005-2006, Bob Ippolito
+# Copyright (c) 2007-2010, Linden Research, Inc.
+# Copyright (c) 2008-2010, Eventlet Contributors (see AUTHORS)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from __future__ import absolute_import, print_function, division
+
+import time
+import re
+import os
+import sys
+
+import _socket
+from _socket import AI_NUMERICHOST
+from _socket import error
+from _socket import NI_NUMERICSERV
+from _socket import AF_INET
+from _socket import AF_INET6
+from _socket import AF_UNSPEC
+
+import socket
+
+from . import AbstractResolver
+from . import hostname_types
+
+from gevent._compat import string_types
+from gevent._compat import iteritems
+from gevent._patcher import import_patched
+from gevent._config import config
+
+__all__ = [
+    'Resolver',
+]
+
+# Import the DNS packages to use the gevent modules,
+# even if the system is not monkey-patched.
+def _patch_dns():
+    top = import_patched('dns')
+    for pkg in ('dns',
+                'dns.rdtypes',
+                'dns.rdtypes.IN',
+                'dns.rdtypes.ANY'):
+        mod = import_patched(pkg)
+        for name in mod.__all__:
+            setattr(mod, name, import_patched(pkg + '.' + name))
+    return top
+
+dns = _patch_dns()
+
+def _dns_import_patched(name):
+    assert name.startswith('dns')
+    import_patched(name)
+    return dns
+
+# This module tries to dynamically import classes
+# using __import__, and it's important that they match
+# the ones we just created, otherwise exceptions won't be caught
+# as expected. It uses a one-arg __import__ statement and then
+# tries to walk down the sub-modules using getattr, so we can't
+# directly use import_patched as-is.
+dns.rdata.__import__ = _dns_import_patched
+
+resolver = dns.resolver
+dTimeout = dns.resolver.Timeout
+
+_exc_clear = getattr(sys, 'exc_clear', lambda: None)
+
+# This is a copy of resolver._getaddrinfo with the crucial change that it
+# doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
+# A secondary change is that calls to sys.exc_clear() have been inserted to avoid
+# failing tests in test__refcount.py (timeouts).
+# See https://github.com/rthalley/dnspython/pull/300
+def _getaddrinfo(host=None, service=None, family=AF_UNSPEC, socktype=0,
+                 proto=0, flags=0):
+    # pylint:disable=too-many-locals,broad-except,too-many-statements
+    # pylint:disable=too-many-branches
+    # pylint:disable=redefined-argument-from-local
+    # pylint:disable=consider-using-in
+    if flags & (socket.AI_ADDRCONFIG | socket.AI_V4MAPPED) != 0:
+        raise NotImplementedError
+    if host is None and service is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    v6addrs = []
+    v4addrs = []
+    canonical_name = None
+    try:
+        # Is host None or a V6 address literal?
+        if host is None:
+            canonical_name = 'localhost'
+            if flags & socket.AI_PASSIVE != 0:
+                v6addrs.append('::')
+                v4addrs.append('0.0.0.0')
+            else:
+                v6addrs.append('::1')
+                v4addrs.append('127.0.0.1')
+        else:
+            parts = host.split('%')
+            if len(parts) == 2:
+                ahost = parts[0]
+            else:
+                ahost = host
+            addr = dns.ipv6.inet_aton(ahost)
+            v6addrs.append(host)
+            canonical_name = host
+    except Exception:
+        _exc_clear()
+        try:
+            # Is it a V4 address literal?
+            addr = dns.ipv4.inet_aton(host)
+            v4addrs.append(host)
+            canonical_name = host
+        except Exception:
+            _exc_clear()
+            if flags & socket.AI_NUMERICHOST == 0:
+                try:
+                    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+                        v6 = resolver._resolver.query(host, dns.rdatatype.AAAA,
+                                                      raise_on_no_answer=False)
+                        # Note that setting host ensures we query the same name
+                        # for A as we did for AAAA.
+                        host = v6.qname
+                        canonical_name = v6.canonical_name.to_text(True)
+                        if v6.rrset is not None:
+                            for rdata in v6.rrset:
+                                v6addrs.append(rdata.address)
+                    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+                        v4 = resolver._resolver.query(host, dns.rdatatype.A,
+                                                      raise_on_no_answer=False)
+                        host = v4.qname
+                        canonical_name = v4.canonical_name.to_text(True)
+                        if v4.rrset is not None:
+                            for rdata in v4.rrset:
+                                v4addrs.append(rdata.address)
+                except dns.resolver.NXDOMAIN:
+                    _exc_clear()
+                    raise socket.gaierror(socket.EAI_NONAME)
+                except Exception:
+                    _exc_clear()
+                    raise socket.gaierror(socket.EAI_SYSTEM)
+    port = None
+    try:
+        # Is it a port literal?
+        if service is None:
+            port = 0
+        else:
+            port = int(service)
+    except Exception:
+        _exc_clear()
+        if flags & socket.AI_NUMERICSERV == 0:
+            try:
+                port = socket.getservbyname(service)
+            except Exception:
+                _exc_clear()
+
+    if port is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    tuples = []
+    if socktype == 0:
+        socktypes = [socket.SOCK_DGRAM, socket.SOCK_STREAM]
+    else:
+        socktypes = [socktype]
+    if flags & socket.AI_CANONNAME != 0:
+        cname = canonical_name
+    else:
+        cname = ''
+    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+        for addr in v6addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET6, socktype, proto,
+                                   cname, (addr, port, 0, 0))) # XXX: gevent: this can get the scopeid wrong
+    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+        for addr in v4addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET, socktype, proto,
+                                   cname, (addr, port)))
+    if len(tuples) == 0: # pylint:disable=len-as-condition
+        raise socket.gaierror(socket.EAI_NONAME)
+    return tuples
+
+
+resolver._getaddrinfo = _getaddrinfo
+
+HOSTS_TTL = 300.0
+
+def _is_addr(host, parse=dns.ipv4.inet_aton):
+    if not host:
+        return False
+    assert isinstance(host, hostname_types), repr(host)
+    try:
+        parse(host)
+    except dns.exception.SyntaxError:
+        return False
+    else:
+        return True
+
+# Return True if host is a valid IPv4 address
+_is_ipv4_addr = _is_addr
+
+
+def _is_ipv6_addr(host):
+    # Return True if host is a valid IPv6 address
+    if host:
+        s = '%' if isinstance(host, str) else b'%'
+        host = host.split(s, 1)[0]
+    return _is_addr(host, dns.ipv6.inet_aton)
+
+class HostsFile(object):
+    """
+    A class to read the contents of a hosts file (/etc/hosts).
+    """
+
+    LINES_RE = re.compile(r"""
+        \s*  # Leading space
+        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
+        \s*  # Trailing space
+        (?:[#][^\r\n]+)?  # Comments
+        (?:$|[\r\n]+)  # EOF or newline
+    """, re.VERBOSE)
+
+    def __init__(self, fname=None):
+        self.v4 = {} # name -> ipv4
+        self.v6 = {} # name -> ipv6
+        self.aliases = {} # name -> canonical_name
+        self.reverse = {} # ip addr -> some name
+        if fname is None:
+            if os.name == 'posix':
+                fname = '/etc/hosts'
+            elif os.name == 'nt': # pragma: no cover
+                fname = os.path.expandvars(
+                    r'%SystemRoot%\system32\drivers\etc\hosts')
+        self.fname = fname
+        assert self.fname
+        self._last_load = 0
+
+
+    def _readlines(self):
+        # Read the contents of the hosts file.
+        #
+        # Return list of lines, comment lines and empty lines are
+        # excluded. Note that this performs disk I/O so can be
+        # blocking.
+        with open(self.fname, 'rb') as fp:
+            fdata = fp.read()
+
+
+        # XXX: Using default decoding. Is that correct?
+        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
+
+        return self.LINES_RE.findall(udata)
+
+    def load(self): # pylint:disable=too-many-locals
+        # Load hosts file
+
+        # This will (re)load the data from the hosts
+        # file if it has changed.
+
+        try:
+            load_time = os.stat(self.fname).st_mtime
+            needs_load = load_time > self._last_load
+        except (IOError, OSError):
+            from gevent import get_hub
+            get_hub().handle_error(self, *sys.exc_info())
+            needs_load = False
+
+        if not needs_load:
+            return
+
+        v4 = {}
+        v6 = {}
+        aliases = {}
+        reverse = {}
+
+        for line in self._readlines():
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            ip = parts.pop(0)
+            if _is_ipv4_addr(ip):
+                ipmap = v4
+            elif _is_ipv6_addr(ip):
+                if ip.startswith('fe80'):
+                    # Do not use link-local addresses, OSX stores these here
+                    continue
+                ipmap = v6
+            else:
+                continue
+            cname = parts.pop(0).lower()
+            ipmap[cname] = ip
+            for alias in parts:
+                alias = alias.lower()
+                ipmap[alias] = ip
+                aliases[alias] = cname
+
+            # XXX: This is wrong for ipv6
+            if ipmap is v4:
+                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
+            else:
+                ptr = ip + '.ip6.arpa.'
+            if ptr not in reverse:
+                reverse[ptr] = cname
+
+        self._last_load = load_time
+        self.v4 = v4
+        self.v6 = v6
+        self.aliases = aliases
+        self.reverse = reverse
+
+    def iter_all_host_addr_pairs(self):
+        self.load()
+        for name, addr in iteritems(self.v4):
+            yield name, addr
+        for name, addr in iteritems(self.v6):
+            yield name, addr
+
+class _HostsAnswer(dns.resolver.Answer):
+    # Answer class for HostsResolver object
+
+    def __init__(self, qname, rdtype, rdclass, rrset, raise_on_no_answer=True):
+        self.response = None
+        self.qname = qname
+        self.rdtype = rdtype
+        self.rdclass = rdclass
+        self.canonical_name = qname
+        if not rrset and raise_on_no_answer:
+            raise dns.resolver.NoAnswer()
+        self.rrset = rrset
+        self.expiration = (time.time() +
+                           rrset.ttl if hasattr(rrset, 'ttl') else 0)
+
+
+class _HostsResolver(object):
+    """
+    Class to parse the hosts file
+    """
+
+    def __init__(self, fname=None, interval=HOSTS_TTL):
+        self.hosts_file = HostsFile(fname)
+        self.interval = interval
+        self._last_load = 0
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True): # pylint:disable=unused-argument
+        # Query the hosts file
+        #
+        # The known rdtypes are dns.rdatatype.A, dns.rdatatype.AAAA and
+        # dns.rdatatype.CNAME.
+        # The ``rdclass`` parameter must be dns.rdataclass.IN while the
+        # ``tcp`` and ``source`` parameters are ignored.
+        # Return a HostAnswer instance or raise a dns.resolver.NoAnswer
+        # exception.
+
+        now = time.time()
+        hosts_file = self.hosts_file
+        if self._last_load + self.interval < now:
+            self._last_load = now
+            hosts_file.load()
+
+        rdclass = dns.rdataclass.IN # Always
+        if isinstance(qname, string_types):
+            name = qname
+            qname = dns.name.from_text(qname)
+        else:
+            name = str(qname)
+
+        name = name.lower()
+        rrset = dns.rrset.RRset(qname, rdclass, rdtype)
+        rrset.ttl = self._last_load + self.interval - now
+
+        if rdtype == dns.rdatatype.A:
+            mapping = hosts_file.v4
+            kind = dns.rdtypes.IN.A.A
+        elif rdtype == dns.rdatatype.AAAA:
+            mapping = hosts_file.v6
+            kind = dns.rdtypes.IN.AAAA.AAAA
+        elif rdtype == dns.rdatatype.CNAME:
+            mapping = hosts_file.aliases
+            kind = lambda c, t, addr: dns.rdtypes.ANY.CNAME.CNAME(c, t, dns.name.from_text(addr))
+        elif rdtype == dns.rdatatype.PTR:
+            mapping = hosts_file.reverse
+            kind = lambda c, t, addr: dns.rdtypes.ANY.PTR.PTR(c, t, dns.name.from_text(addr))
+
+
+        addr = mapping.get(name)
+        if not addr and qname.is_absolute():
+            addr = mapping.get(name[:-1])
+        if addr:
+            rrset.add(kind(rdclass, rdtype, addr))
+        return _HostsAnswer(qname, rdtype, rdclass, rrset, raise_on_no_answer)
+
+    def getaliases(self, hostname):
+        # Return a list of all the aliases of a given cname
+
+        # Due to the way store aliases this is a bit inefficient, this
+        # clearly was an afterthought.  But this is only used by
+        # gethostbyname_ex so it's probably fine.
+        aliases = self.hosts_file.aliases
+        result = []
+        if hostname in aliases:
+            cannon = aliases[hostname]
+        else:
+            cannon = hostname
+        result.append(cannon)
+        for alias, cname in iteritems(aliases):
+            if cannon == cname:
+                result.append(alias)
+        result.remove(hostname)
+        return result
+
+class _DualResolver(object):
+
+    def __init__(self):
+        self.hosts_resolver = _HostsResolver()
+        self.network_resolver = resolver.get_default_resolver()
+        self.network_resolver.cache = resolver.LRUCache()
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True,
+              _hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)):
+        # Query the resolver, using /etc/hosts
+
+        # Behavior:
+        # 1. if hosts is enabled and contains answer, return it now
+        # 2. query nameservers for qname
+        if qname is None:
+            qname = '0.0.0.0'
+
+        if not isinstance(qname, string_types):
+            if isinstance(qname, bytes):
+                qname = qname.decode("idna")
+
+        if isinstance(qname, string_types):
+            qname = dns.name.from_text(qname, None)
+
+        if isinstance(rdtype, string_types):
+            rdtype = dns.rdatatype.from_text(rdtype)
+
+        if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes:
+            try:
+                answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False)
+            except Exception: # pylint: disable=broad-except
+                from gevent import get_hub
+                get_hub().handle_error(self, *sys.exc_info())
+            else:
+                if answer.rrset:
+                    return answer
+
+        return self.network_resolver.query(qname, rdtype, rdclass,
+                                           tcp, source, raise_on_no_answer=raise_on_no_answer)
+
+def _family_to_rdtype(family):
+    if family == socket.AF_INET:
+        rdtype = dns.rdatatype.A
+    elif family == socket.AF_INET6:
+        rdtype = dns.rdatatype.AAAA
+    else:
+        raise socket.gaierror(socket.EAI_FAMILY,
+                              'Address family not supported')
+    return rdtype
+
+class Resolver(AbstractResolver):
+    """
+    An *experimental* resolver that uses `dnspython`_.
+
+    This is typically slower than the default threaded resolver
+    (unless there's a cache hit, in which case it can be much faster).
+    It is usually much faster than the c-ares resolver. It tends to
+    scale well as more concurrent resolutions are attempted.
+
+    Under Python 2, if the ``idna`` package is installed, this
+    resolver can resolve Unicode host names that the system resolver
+    cannot.
+
+    .. note::
+
+        This **does not** use dnspython's default resolver object, or share any
+        classes with ``import dns``. A separate copy of the objects is imported to
+        be able to function in a non monkey-patched process. The documentation for the resolver
+        object still applies.
+
+        The resolver that we use is available as the :attr:`resolver` attribute
+        of this object (typically ``gevent.get_hub().resolver.resolver``).
+
+    .. caution::
+
+        Many of the same caveats about DNS results apply here as are documented
+        for :class:`gevent.resolver.ares.Resolver`.
+
+    .. caution::
+
+        This resolver is experimental. It may be removed or modified in
+        the future. As always, feedback is welcome.
+
+    .. versionadded:: 1.3a2
+
+    .. _dnspython: http://www.dnspython.org
+    """
+
+    def __init__(self, hub=None): # pylint: disable=unused-argument
+        if resolver._resolver is None:
+            _resolver = resolver._resolver = _DualResolver()
+            if config.resolver_nameservers:
+                _resolver.network_resolver.nameservers[:] = config.resolver_nameservers
+            if config.resolver_timeout:
+                _resolver.network_resolver.lifetime = config.resolver_timeout
+        # Different hubs in different threads could be sharing the same
+        # resolver.
+        assert isinstance(resolver._resolver, _DualResolver)
+        self._resolver = resolver._resolver
+
+    @property
+    def resolver(self):
+        """
+        The dnspython resolver object we use.
+
+        This object has several useful attributes that can be used to
+        adjust the behaviour of the DNS system:
+
+        * ``cache`` is a :class:`dns.resolver.LRUCache`. Its maximum size
+          can be configured by calling :meth:`resolver.cache.set_max_size`
+        * ``nameservers`` controls which nameservers to talk to
+        * ``lifetime`` configures a timeout for each individual query.
+        """
+        return self._resolver.network_resolver
+
+    def close(self):
+        pass
+
+    def _getaliases(self, hostname, family):
+        if not isinstance(hostname, str):
+            if isinstance(hostname, bytes):
+                hostname = hostname.decode("idna")
+        aliases = self._resolver.hosts_resolver.getaliases(hostname)
+        net_resolver = self._resolver.network_resolver
+        rdtype = _family_to_rdtype(family)
+        while True:
+            try:
+                ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
+            except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
+                break
+            except dTimeout:
+                break
+            else:
+                aliases.extend(str(rr.target) for rr in ans.rrset)
+                hostname = ans[0].target
+        return aliases
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        if ((host in (u'localhost', b'localhost')
+             or (_is_ipv6_addr(host) and host.startswith('fe80')))
+                or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
+            # this handles cases which do not require network access
+            # 1) host is None
+            # 2) host is of an invalid type
+            # 3) host is localhost or a link-local ipv6; dnspython returns the wrong
+            #    scope-id for those.
+            # 3) AI_NUMERICHOST flag is set
+
+            return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
+
+        if family == AF_UNSPEC:
+            # This tends to raise in the case that a v6 address did not exist
+            # but a v4 does. So we break it into two parts.
+
+            # Note that if there is no ipv6 in the hosts file, but there *is*
+            # an ipv4, and there *is* an ipv6 in the nameservers, we will return
+            # both (from the first call). The system resolver on OS X only returns
+            # the results from the hosts file. doubleclick.com is one example.
+
+            # See also https://github.com/gevent/gevent/issues/1012
+            try:
+                return _getaddrinfo(host, port, family, socktype, proto, flags)
+            except socket.gaierror:
+                try:
+                    return _getaddrinfo(host, port, AF_INET6, socktype, proto, flags)
+                except socket.gaierror:
+                    return _getaddrinfo(host, port, AF_INET, socktype, proto, flags)
+        else:
+            return _getaddrinfo(host, port, family, socktype, proto, flags)
+
+    def getnameinfo(self, sockaddr, flags):
+        if (sockaddr
+                and isinstance(sockaddr, (list, tuple))
+                and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
+            return _socket.getnameinfo(sockaddr, flags)
+        if isinstance(sockaddr, (list, tuple)) and not isinstance(sockaddr[0], hostname_types):
+            raise TypeError("getnameinfo(): illegal sockaddr argument")
+        try:
+            return resolver._getnameinfo(sockaddr, flags)
+        except error:
+            if not flags:
+                # dnspython doesn't like getting ports it can't resolve.
+                # We have one test, test__socket_dns.py:Test_getnameinfo_geventorg.test_port_zero
+                # that does this. We conservatively fix it here; this could be expanded later.
+                return resolver._getnameinfo(sockaddr, NI_NUMERICSERV)
+
+    def gethostbyaddr(self, ip_address):
+        if ip_address in (u'127.0.0.1', u'::1',
+                          b'127.0.0.1', b'::1',
+                          'localhost'):
+            return _socket.gethostbyaddr(ip_address)
+
+        if not isinstance(ip_address, hostname_types):
+            raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(ip_address),))
+
+        return resolver._gethostbyaddr(ip_address)

python/gevent/resolver/libcares.pxd

@@ -0,0 +1,109 @@
+cdef extern from "ares.h":
+    struct ares_options:
+        int flags
+        void* sock_state_cb
+        void* sock_state_cb_data
+        int timeout
+        int tries
+        int ndots
+        unsigned short udp_port
+        unsigned short tcp_port
+        char **domains
+        int ndomains
+        char* lookups
+
+    int ARES_OPT_FLAGS
+    int ARES_OPT_SOCK_STATE_CB
+    int ARES_OPT_TIMEOUTMS
+    int ARES_OPT_TRIES
+    int ARES_OPT_NDOTS
+    int ARES_OPT_TCP_PORT
+    int ARES_OPT_UDP_PORT
+    int ARES_OPT_SERVERS
+    int ARES_OPT_DOMAINS
+    int ARES_OPT_LOOKUPS
+
+    int ARES_FLAG_USEVC
+    int ARES_FLAG_PRIMARY
+    int ARES_FLAG_IGNTC
+    int ARES_FLAG_NORECURSE
+    int ARES_FLAG_STAYOPEN
+    int ARES_FLAG_NOSEARCH
+    int ARES_FLAG_NOALIASES
+    int ARES_FLAG_NOCHECKRESP
+
+    int ARES_LIB_INIT_ALL
+    int ARES_SOCKET_BAD
+
+    int ARES_SUCCESS
+    int ARES_ENODATA
+    int ARES_EFORMERR
+    int ARES_ESERVFAIL
+    int ARES_ENOTFOUND
+    int ARES_ENOTIMP
+    int ARES_EREFUSED
+    int ARES_EBADQUERY
+    int ARES_EBADNAME
+    int ARES_EBADFAMILY
+    int ARES_EBADRESP
+    int ARES_ECONNREFUSED
+    int ARES_ETIMEOUT
+    int ARES_EOF
+    int ARES_EFILE
+    int ARES_ENOMEM
+    int ARES_EDESTRUCTION
+    int ARES_EBADSTR
+    int ARES_EBADFLAGS
+    int ARES_ENONAME
+    int ARES_EBADHINTS
+    int ARES_ENOTINITIALIZED
+    int ARES_ELOADIPHLPAPI
+    int ARES_EADDRGETNETWORKPARAMS
+    int ARES_ECANCELLED
+
+    int ARES_NI_NOFQDN
+    int ARES_NI_NUMERICHOST
+    int ARES_NI_NAMEREQD
+    int ARES_NI_NUMERICSERV
+    int ARES_NI_DGRAM
+    int ARES_NI_TCP
+    int ARES_NI_UDP
+    int ARES_NI_SCTP
+    int ARES_NI_DCCP
+    int ARES_NI_NUMERICSCOPE
+    int ARES_NI_LOOKUPHOST
+    int ARES_NI_LOOKUPSERVICE
+
+
+    int ares_library_init(int flags)
+    void ares_library_cleanup()
+    int ares_init_options(void *channelptr, ares_options *options, int)
+    int ares_init(void *channelptr)
+    void ares_destroy(void *channelptr)
+    void ares_gethostbyname(void* channel, char *name, int family, void* callback, void *arg)
+    void ares_gethostbyaddr(void* channel, void *addr, int addrlen, int family, void* callback, void *arg)
+    void ares_process_fd(void* channel, int read_fd, int write_fd)
+    char* ares_strerror(int code)
+    void ares_cancel(void* channel)
+    void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg)
+
+    struct in_addr:
+        pass
+
+    struct ares_in6_addr:
+        pass
+
+    struct addr_union:
+        in_addr addr4
+        ares_in6_addr addr6
+
+    struct ares_addr_node:
+        ares_addr_node *next
+        int family
+        addr_union addr
+
+    int ares_set_servers(void* channel, ares_addr_node *servers)
+
+
+cdef extern from "cares_pton.h":
+    int ares_inet_pton(int af, char *src, void *dst)

python/gevent/resolver/thread.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details.
+"""
+Native thread-based hostname resolver.
+"""
+import _socket
+
+from gevent.hub import get_hub
+
+
+__all__ = ['Resolver']
+
+
+# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349
+u'foo'.encode('idna')
+
+
+class Resolver(object):
+    """
+    Implementation of the resolver API using native threads and native resolution
+    functions.
+
+    Using the native resolution mechanisms ensures the highest
+    compatibility with what a non-gevent program would return
+    including good support for platform specific configuration
+    mechanisms. The use of native (non-greenlet) threads ensures that
+    a caller doesn't block other greenlets.
+
+    This implementation also has the benefit of being very simple in comparison to
+    :class:`gevent.resolver_ares.Resolver`.
+
+    .. tip::
+
+        Most users find this resolver to be quite reliable in a
+        properly monkey-patched environment. However, there have been
+        some reports of long delays, slow performance or even hangs,
+        particularly in long-lived programs that make many, many DNS
+        requests. If you suspect that may be happening to you, try the
+        dnspython or ares resolver (and submit a bug report).
+    """
+    def __init__(self, hub=None):
+        if hub is None:
+            hub = get_hub()
+        self.pool = hub.threadpool
+        if _socket.gaierror not in hub.NOT_ERROR:
+            # Do not cause lookup failures to get printed by the default
+            # error handler. This can be very noisy.
+            hub.NOT_ERROR += (_socket.gaierror, _socket.herror)
+
+    def __repr__(self):
+        return '<gevent.resolver_thread.Resolver at 0x%x pool=%r>' % (id(self), self.pool)
+
+    def close(self):
+        pass
+
+    # from briefly reading socketmodule.c, it seems that all of the functions
+    # below are thread-safe in Python, even if they are not thread-safe in C.
+
+    def gethostbyname(self, *args):
+        return self.pool.apply(_socket.gethostbyname, args)
+
+    def gethostbyname_ex(self, *args):
+        return self.pool.apply(_socket.gethostbyname_ex, args)
+
+    def getaddrinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getaddrinfo, args, kwargs)
+
+    def gethostbyaddr(self, *args, **kwargs):
+        return self.pool.apply(_socket.gethostbyaddr, args, kwargs)
+
+    def getnameinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getnameinfo, args, kwargs)