heckyel/yt-local
0
0
Fork 0
Code Issues Pull Requests Releases Activity

Windows: Use 32-bit distribution of python

Browse Source
This commit is contained in:
James Taylor
2018-09-14 19:32:27 -07:00
parent 6ca20ff701
commit 4212164e91
166 changed files with 175548 additions and 44620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
python/gevent/libuv/__init__.py Normal file
View File

BIN
python/gevent/libuv/_corecffi.cp36-win32.pyd Normal file
View File

Binary file not shown.

253
python/gevent/libuv/_corecffi_build.py Normal file
View File

@@ -0,0 +1,253 @@
# pylint: disable=no-member
# This module is only used to create and compile the gevent._corecffi module;
# nothing should be directly imported from it except `ffi`, which should only be
# used for `ffi.compile()`; programs should import gevent._corecfffi.
# However, because we are using "out-of-line" mode, it is necessary to examine
# this file to know what functions are created and available on the generated
# module.
from __future__ import absolute_import, print_function
import sys
import os
import os.path # pylint:disable=no-name-in-module
import struct
__all__ = []
WIN = sys.platform.startswith('win32')
def system_bits():
return struct.calcsize('P') * 8
def st_nlink_type():
if sys.platform == "darwin" or sys.platform.startswith("freebsd"):
return "short"
if system_bits() == 32:
return "unsigned long"
return "long long"
from cffi import FFI
ffi = FFI()
thisdir = os.path.dirname(os.path.abspath(__file__))
def read_source(name):
with open(os.path.join(thisdir, name), 'r') as f:
return f.read()
_cdef = read_source('_corecffi_cdef.c')
_source = read_source('_corecffi_source.c')
_cdef = _cdef.replace('#define GEVENT_ST_NLINK_T int', '')
_cdef = _cdef.replace('#define GEVENT_STRUCT_DONE int', '')
_cdef = _cdef.replace('#define GEVENT_UV_OS_SOCK_T int', '')
_cdef = _cdef.replace('GEVENT_ST_NLINK_T', st_nlink_type())
_cdef = _cdef.replace("GEVENT_STRUCT_DONE _;", '...;')
# uv_os_sock_t is int on POSIX and SOCKET on Win32, but socket is
# just another name for handle, which is just another name for 'void*'
# which we will treat as an 'unsigned long' or 'unsigned long long'
# since it comes through 'fileno()' where it has been cast as an int.
# See class watcher.io
_void_pointer_as_integer = 'intptr_t'
_cdef = _cdef.replace("GEVENT_UV_OS_SOCK_T", 'int' if not WIN else _void_pointer_as_integer)
setup_py_dir = os.path.abspath(os.path.join(thisdir, '..', '..', '..'))
libuv_dir = os.path.abspath(os.path.join(setup_py_dir, 'deps', 'libuv'))
LIBUV_INCLUDE_DIRS = [
thisdir, # libev_vfd.h
os.path.join(libuv_dir, 'include'),
os.path.join(libuv_dir, 'src'),
]
# Initially based on https://github.com/saghul/pyuv/blob/v1.x/setup_libuv.py
def _libuv_source(rel_path):
# Certain versions of setuptools, notably on windows, are *very*
# picky about what we feed to sources= "setup() arguments must
# *always* be /-separated paths relative to the setup.py
# directory, *never* absolute paths." POSIX doesn't have that issue.
path = os.path.join('deps', 'libuv', 'src', rel_path)
return path
LIBUV_SOURCES = [
_libuv_source('fs-poll.c'),
_libuv_source('inet.c'),
_libuv_source('threadpool.c'),
_libuv_source('uv-common.c'),
_libuv_source('version.c'),
_libuv_source('uv-data-getter-setters.c'),
_libuv_source('timer.c'),
]
if WIN:
LIBUV_SOURCES += [
_libuv_source('win/async.c'),
_libuv_source('win/core.c'),
_libuv_source('win/detect-wakeup.c'),
_libuv_source('win/dl.c'),
_libuv_source('win/error.c'),
_libuv_source('win/fs-event.c'),
_libuv_source('win/fs.c'),
# getaddrinfo.c refers to ConvertInterfaceIndexToLuid
# and ConvertInterfaceLuidToNameA, which are supposedly in iphlpapi.h
# and iphlpapi.lib/dll. But on Windows 10 with Python 3.5 and VC 14 (Visual Studio 2015),
# I get an undefined warning from the compiler for those functions and
# a link error from the linker, so this file can't be included.
# This is possibly because the functions are defined for Windows Vista, and
# Python 3.5 builds with at earlier SDK?
# Fortunately we don't use those functions.
#_libuv_source('win/getaddrinfo.c'),
# getnameinfo.c refers to uv__getaddrinfo_translate_error from
# getaddrinfo.c, which we don't have.
#_libuv_source('win/getnameinfo.c'),
_libuv_source('win/handle.c'),
_libuv_source('win/loop-watcher.c'),
_libuv_source('win/pipe.c'),
_libuv_source('win/poll.c'),
_libuv_source('win/process-stdio.c'),
_libuv_source('win/process.c'),
_libuv_source('win/req.c'),
_libuv_source('win/signal.c'),
_libuv_source('win/snprintf.c'),
_libuv_source('win/stream.c'),
_libuv_source('win/tcp.c'),
_libuv_source('win/thread.c'),
_libuv_source('win/tty.c'),
_libuv_source('win/udp.c'),
_libuv_source('win/util.c'),
_libuv_source('win/winapi.c'),
_libuv_source('win/winsock.c'),
]
else:
LIBUV_SOURCES += [
_libuv_source('unix/async.c'),
_libuv_source('unix/core.c'),
_libuv_source('unix/dl.c'),
_libuv_source('unix/fs.c'),
_libuv_source('unix/getaddrinfo.c'),
_libuv_source('unix/getnameinfo.c'),
_libuv_source('unix/loop-watcher.c'),
_libuv_source('unix/loop.c'),
_libuv_source('unix/pipe.c'),
_libuv_source('unix/poll.c'),
_libuv_source('unix/process.c'),
_libuv_source('unix/signal.c'),
_libuv_source('unix/stream.c'),
_libuv_source('unix/tcp.c'),
_libuv_source('unix/thread.c'),
_libuv_source('unix/tty.c'),
_libuv_source('unix/udp.c'),
]
if sys.platform.startswith('linux'):
LIBUV_SOURCES += [
_libuv_source('unix/linux-core.c'),
_libuv_source('unix/linux-inotify.c'),
_libuv_source('unix/linux-syscalls.c'),
_libuv_source('unix/procfs-exepath.c'),
_libuv_source('unix/proctitle.c'),
_libuv_source('unix/sysinfo-loadavg.c'),
_libuv_source('unix/sysinfo-memory.c'),
]
elif sys.platform == 'darwin':
LIBUV_SOURCES += [
_libuv_source('unix/bsd-ifaddrs.c'),
_libuv_source('unix/darwin.c'),
_libuv_source('unix/darwin-proctitle.c'),
_libuv_source('unix/fsevents.c'),
_libuv_source('unix/kqueue.c'),
_libuv_source('unix/proctitle.c'),
]
elif sys.platform.startswith(('freebsd', 'dragonfly')):
LIBUV_SOURCES += [
_libuv_source('unix/bsd-ifaddrs.c'),
_libuv_source('unix/freebsd.c'),
_libuv_source('unix/kqueue.c'),
_libuv_source('unix/posix-hrtime.c'),
]
elif sys.platform.startswith('openbsd'):
LIBUV_SOURCES += [
_libuv_source('unix/bsd-ifaddrs.c'),
_libuv_source('unix/kqueue.c'),
_libuv_source('unix/openbsd.c'),
_libuv_source('unix/posix-hrtime.c'),
]
elif sys.platform.startswith('netbsd'):
LIBUV_SOURCES += [
_libuv_source('unix/bsd-ifaddrs.c'),
_libuv_source('unix/kqueue.c'),
_libuv_source('unix/netbsd.c'),
_libuv_source('unix/posix-hrtime.c'),
]
elif sys.platform.startswith('sunos'):
LIBUV_SOURCES += [
_libuv_source('unix/no-proctitle.c'),
_libuv_source('unix/sunos.c'),
]
LIBUV_MACROS = []
def _define_macro(name, value):
LIBUV_MACROS.append((name, value))
LIBUV_LIBRARIES = []
def _add_library(name):
LIBUV_LIBRARIES.append(name)
if sys.platform != 'win32':
_define_macro('_LARGEFILE_SOURCE', 1)
_define_macro('_FILE_OFFSET_BITS', 64)
if sys.platform.startswith('linux'):
_add_library('dl')
_add_library('rt')
_define_macro('_GNU_SOURCE', 1)
_define_macro('_POSIX_C_SOURCE', '200112')
elif sys.platform == 'darwin':
_define_macro('_DARWIN_USE_64_BIT_INODE', 1)
_define_macro('_DARWIN_UNLIMITED_SELECT', 1)
elif sys.platform.startswith('netbsd'):
_add_library('kvm')
elif sys.platform.startswith('sunos'):
_define_macro('__EXTENSIONS__', 1)
_define_macro('_XOPEN_SOURCE', 500)
_add_library('kstat')
_add_library('nsl')
_add_library('sendfile')
_add_library('socket')
elif WIN:
_define_macro('_GNU_SOURCE', 1)
_define_macro('WIN32', 1)
_define_macro('_CRT_SECURE_NO_DEPRECATE', 1)
_define_macro('_CRT_NONSTDC_NO_DEPRECATE', 1)
_define_macro('_CRT_SECURE_NO_WARNINGS', 1)
_define_macro('_WIN32_WINNT', '0x0600')
_define_macro('WIN32_LEAN_AND_MEAN', 1)
_add_library('advapi32')
_add_library('iphlpapi')
_add_library('psapi')
_add_library('shell32')
_add_library('user32')
_add_library('userenv')
_add_library('ws2_32')
ffi.cdef(_cdef)
ffi.set_source('gevent.libuv._corecffi',
_source,
sources=LIBUV_SOURCES,
depends=LIBUV_SOURCES,
include_dirs=LIBUV_INCLUDE_DIRS,
libraries=list(LIBUV_LIBRARIES),
define_macros=list(LIBUV_MACROS))
if __name__ == '__main__':
ffi.compile()

393
python/gevent/libuv/_corecffi_cdef.c Normal file
View File

@@ -0,0 +1,393 @@
/* markers for the CFFI parser. Replaced when the string is read. */
#define GEVENT_STRUCT_DONE int
#define GEVENT_ST_NLINK_T int
#define GEVENT_UV_OS_SOCK_T int
#define UV_EBUSY ...
#define UV_VERSION_MAJOR ...
#define UV_VERSION_MINOR ...
#define UV_VERSION_PATCH ...
typedef enum {
UV_RUN_DEFAULT = 0,
UV_RUN_ONCE,
UV_RUN_NOWAIT
} uv_run_mode;
typedef enum {
UV_UNKNOWN_HANDLE = 0,
UV_ASYNC,
UV_CHECK,
UV_FS_EVENT,
UV_FS_POLL,
UV_HANDLE,
UV_IDLE,
UV_NAMED_PIPE,
UV_POLL,
UV_PREPARE,
UV_PROCESS,
UV_STREAM,
UV_TCP,
UV_TIMER,
UV_TTY,
UV_UDP,
UV_SIGNAL,
UV_FILE,
UV_HANDLE_TYPE_MAX
} uv_handle_type;
enum uv_poll_event {
UV_READABLE = 1,
UV_WRITABLE = 2,
/* new in 1.9 */
UV_DISCONNECT = 4,
/* new in 1.14.0 */
UV_PRIORITIZED = 8,
};
enum uv_fs_event {
UV_RENAME = 1,
UV_CHANGE = 2
};
enum uv_fs_event_flags {
/*
* By default, if the fs event watcher is given a directory name, we will
* watch for all events in that directory. This flags overrides this behavior
* and makes fs_event report only changes to the directory entry itself. This
* flag does not affect individual files watched.
* This flag is currently not implemented yet on any backend.
*/
UV_FS_EVENT_WATCH_ENTRY = 1,
/*
* By default uv_fs_event will try to use a kernel interface such as inotify
* or kqueue to detect events. This may not work on remote filesystems such
* as NFS mounts. This flag makes fs_event fall back to calling stat() on a
* regular interval.
* This flag is currently not implemented yet on any backend.
*/
UV_FS_EVENT_STAT = 2,
/*
* By default, event watcher, when watching directory, is not registering
* (is ignoring) changes in it's subdirectories.
* This flag will override this behaviour on platforms that support it.
*/
UV_FS_EVENT_RECURSIVE = 4
};
const char* uv_strerror(int);
const char* uv_err_name(int);
const char* uv_version_string(void);
const char* uv_handle_type_name(uv_handle_type type);
// handle structs and types
struct uv_loop_s {
void* data;
GEVENT_STRUCT_DONE _;
};
struct uv_handle_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_idle_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_prepare_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_timer_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_signal_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_poll_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_check_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_async_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
void (*async_cb)(struct uv_async_s *);
GEVENT_STRUCT_DONE _;
};
struct uv_fs_event_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
struct uv_fs_poll_s {
struct uv_loop_s* loop;
uv_handle_type type;
void *data;
GEVENT_STRUCT_DONE _;
};
typedef struct uv_loop_s uv_loop_t;
typedef struct uv_handle_s uv_handle_t;
typedef struct uv_idle_s uv_idle_t;
typedef struct uv_prepare_s uv_prepare_t;
typedef struct uv_timer_s uv_timer_t;
typedef struct uv_signal_s uv_signal_t;
typedef struct uv_poll_s uv_poll_t;
typedef struct uv_check_s uv_check_t;
typedef struct uv_async_s uv_async_t;
typedef struct uv_fs_event_s uv_fs_event_t;
typedef struct uv_fs_poll_s uv_fs_poll_t;
size_t uv_handle_size(uv_handle_type);
// callbacks with the same signature
typedef void (*uv_close_cb)(uv_handle_t *handle);
typedef void (*uv_idle_cb)(uv_idle_t *handle);
typedef void (*uv_timer_cb)(uv_timer_t *handle);
typedef void (*uv_check_cb)(uv_check_t* handle);
typedef void (*uv_async_cb)(uv_async_t* handle);
typedef void (*uv_prepare_cb)(uv_prepare_t *handle);
// callbacks with distinct sigs
typedef void (*uv_walk_cb)(uv_handle_t *handle, void *arg);
typedef void (*uv_poll_cb)(uv_poll_t *handle, int status, int events);
typedef void (*uv_signal_cb)(uv_signal_t *handle, int signum);
// Callback passed to uv_fs_event_start() which will be called
// repeatedly after the handle is started. If the handle was started
// with a directory the filename parameter will be a relative path to
// a file contained in the directory. The events parameter is an ORed
// mask of uv_fs_event elements.
typedef void (*uv_fs_event_cb)(uv_fs_event_t* handle, const char* filename, int events, int status);
typedef struct {
long tv_sec;
long tv_nsec;
} uv_timespec_t;
typedef struct {
uint64_t st_dev;
uint64_t st_mode;
uint64_t st_nlink;
uint64_t st_uid;
uint64_t st_gid;
uint64_t st_rdev;
uint64_t st_ino;
uint64_t st_size;
uint64_t st_blksize;
uint64_t st_blocks;
uint64_t st_flags;
uint64_t st_gen;
uv_timespec_t st_atim;
uv_timespec_t st_mtim;
uv_timespec_t st_ctim;
uv_timespec_t st_birthtim;
} uv_stat_t;
typedef void (*uv_fs_poll_cb)(uv_fs_poll_t* handle, int status, const uv_stat_t* prev, const uv_stat_t* curr);
// loop functions
uv_loop_t *uv_default_loop();
uv_loop_t* uv_loop_new(); // not documented; neither is uv_loop_delete
int uv_loop_init(uv_loop_t* loop);
int uv_loop_fork(uv_loop_t* loop);
int uv_loop_alive(const uv_loop_t *loop);
int uv_loop_close(uv_loop_t* loop);
uint64_t uv_backend_timeout(uv_loop_t* loop);
int uv_run(uv_loop_t *, uv_run_mode mode);
int uv_backend_fd(const uv_loop_t* loop);
// The narrative docs for the two time functions say 'const',
// but the header does not.
void uv_update_time(uv_loop_t* loop);
uint64_t uv_now(uv_loop_t* loop);
void uv_stop(uv_loop_t *);
void uv_walk(uv_loop_t *loop, uv_walk_cb walk_cb, void *arg);
// handle functions
// uv_handle_t is the base type for all libuv handle types.
void uv_ref(void *);
void uv_unref(void *);
int uv_has_ref(void *);
void uv_close(void *handle, uv_close_cb close_cb);
int uv_is_active(void *handle);
int uv_is_closing(void *handle);
// idle functions
// Idle handles will run the given callback once per loop iteration, right
// before the uv_prepare_t handles. Note: The notable difference with prepare
// handles is that when there are active idle handles, the loop will perform a
// zero timeout poll instead of blocking for i/o. Warning: Despite the name,
// idle handles will get their callbacks called on every loop iteration, not
// when the loop is actually "idle".
int uv_idle_init(uv_loop_t *, uv_idle_t *idle);
int uv_idle_start(uv_idle_t *idle, uv_idle_cb cb);
int uv_idle_stop(uv_idle_t *idle);
// prepare functions
// Prepare handles will run the given callback once per loop iteration, right
// before polling for i/o.
int uv_prepare_init(uv_loop_t *, uv_prepare_t *prepare);
int uv_prepare_start(uv_prepare_t *prepare, uv_prepare_cb cb);
int uv_prepare_stop(uv_prepare_t *prepare);
// check functions
// Check handles will run the given callback once per loop iteration, right
int uv_check_init(uv_loop_t *, uv_check_t *check);
int uv_check_start(uv_check_t *check, uv_check_cb cb);
int uv_check_stop(uv_check_t *check);
// async functions
// Async handles allow the user to "wakeup" the event loop and get a callback called from another thread.
int uv_async_init(uv_loop_t *, uv_async_t*, uv_async_cb);
int uv_async_send(uv_async_t*);
// timer functions
// Timer handles are used to schedule callbacks to be called in the future.
int uv_timer_init(uv_loop_t *, uv_timer_t *handle);
int uv_timer_start(uv_timer_t *handle, uv_timer_cb cb, uint64_t timeout, uint64_t repeat);
int uv_timer_stop(uv_timer_t *handle);
int uv_timer_again(uv_timer_t *handle);
void uv_timer_set_repeat(uv_timer_t *handle, uint64_t repeat);
uint64_t uv_timer_get_repeat(const uv_timer_t *handle);
// signal functions
// Signal handles implement Unix style signal handling on a per-event loop
// bases.
int uv_signal_init(uv_loop_t *loop, uv_signal_t *handle);
int uv_signal_start(uv_signal_t *handle, uv_signal_cb signal_cb, int signum);
int uv_signal_stop(uv_signal_t *handle);
// poll functions Poll handles are used to watch file descriptors for
// readability and writability, similar to the purpose of poll(2). It
// is not okay to have multiple active poll handles for the same
// socket, this can cause libuv to busyloop or otherwise malfunction.
//
// The purpose of poll handles is to enable integrating external
// libraries that rely on the event loop to signal it about the socket
// status changes, like c-ares or libssh2. Using uv_poll_t for any
// other purpose is not recommended; uv_tcp_t, uv_udp_t, etc. provide
// an implementation that is faster and more scalable than what can be
// achieved with uv_poll_t, especially on Windows.
//
// Note On windows only sockets can be polled with poll handles. On
// Unix any file descriptor that would be accepted by poll(2) can be
// used.
int uv_poll_init(uv_loop_t *loop, uv_poll_t *handle, int fd);
// Initialize the handle using a socket descriptor. On Unix this is
// identical to uv_poll_init(). On windows it takes a SOCKET handle;
// SOCKET handles are another name for HANDLE objects in win32, and
// those are defined as PVOID, even though they are not actually
// pointers (they're small integers). CPython and PyPy both return
// the SOCKET (as cast to an int) from the socket.fileno() method.
// libuv uses ``uv_os_sock_t`` for this type, which is defined as an
// int on unix.
int uv_poll_init_socket(uv_loop_t* loop, uv_poll_t* handle, GEVENT_UV_OS_SOCK_T socket);
int uv_poll_start(uv_poll_t *handle, int events, uv_poll_cb cb);
int uv_poll_stop(uv_poll_t *handle);
// FS Event handles allow the user to monitor a given path for
// changes, for example, if the file was renamed or there was a
// generic change in it. This handle uses the best backend for the job
// on each platform.
//
// Thereas also uv_fs_poll_t that uses stat for filesystems where
// the kernel event isn't available.
int uv_fs_event_init(uv_loop_t*, uv_fs_event_t*);
int uv_fs_event_start(uv_fs_event_t*, uv_fs_event_cb, const char* path, unsigned int flags);
int uv_fs_event_stop(uv_fs_event_t*);
int uv_fs_event_getpath(uv_fs_event_t*, char* buffer, size_t* size);
// FS Poll handles allow the user to monitor a given path for changes.
// Unlike uv_fs_event_t, fs poll handles use stat to detect when a
// file has changed so they can work on file systems where fs event
// handles can't.
//
// This is a closer match to libev.
int uv_fs_poll_init(void*, void*);
int uv_fs_poll_start(void*, uv_fs_poll_cb, const char* path, unsigned int);
int uv_fs_poll_stop(void*);
/* Standard library */
void* memset(void *b, int c, size_t len);
/* gevent callbacks */
// Implemented in Python code as 'def_extern'. In the case of poll callbacks and fs
// callbacks, if *status* is less than 0, it will be passed in the revents
// field. In cases of no extra arguments, revents will be 0.
// These will be created as static functions at the end of the
// _source.c and must be pre-declared at the top of that file if we
// call them
typedef void* GeventWatcherObject;
extern "Python" {
// Standard gevent._ffi.loop callbacks.
int python_callback(GeventWatcherObject handle, int revents);
void python_handle_error(GeventWatcherObject handle, int revents);
void python_stop(GeventWatcherObject handle);
void python_check_callback(uv_check_t* handle);
void python_prepare_callback(uv_prepare_t* handle);
void python_timer0_callback(uv_check_t* handle);
// libuv specific callback
void _uv_close_callback(uv_handle_t* handle);
void python_sigchld_callback(uv_signal_t* handle, int signum);
void python_queue_callback(uv_handle_t* handle, int revents);
}
// A variable we fill in.
static void (*gevent_noop)(void* handle);
static void _gevent_signal_callback1(uv_signal_t* handle, int arg);
static void _gevent_async_callback0(uv_async_t* handle);
static void _gevent_prepare_callback0(uv_prepare_t* handle);
static void _gevent_timer_callback0(uv_timer_t* handle);
static void _gevent_check_callback0(uv_check_t* handle);
static void _gevent_idle_callback0(uv_idle_t* handle);
static void _gevent_poll_callback2(uv_poll_t* handle, int status, int events);
static void _gevent_fs_event_callback3(uv_fs_event_t* handle, const char* filename, int events, int status);
typedef struct _gevent_fs_poll_s {
uv_fs_poll_t handle;
uv_stat_t curr;
uv_stat_t prev;
} gevent_fs_poll_t;
static void _gevent_fs_poll_callback3(uv_fs_poll_t* handle, int status, const uv_stat_t* prev, const uv_stat_t* curr);
static void gevent_uv_walk_callback_close(uv_handle_t* handle, void* arg);
static void gevent_close_all_handles(uv_loop_t* loop);
static void gevent_zero_timer(uv_timer_t* handle);
static void gevent_zero_prepare(uv_prepare_t* handle);
static void gevent_zero_check(uv_check_t* handle);
static void gevent_zero_loop(uv_loop_t* handle);

181
python/gevent/libuv/_corecffi_source.c Normal file
View File

@@ -0,0 +1,181 @@
#include <string.h>
#include "uv.h"
typedef void* GeventWatcherObject;
static int python_callback(GeventWatcherObject handle, int revents);
static void python_queue_callback(uv_handle_t* watcher_ptr, int revents);
static void python_handle_error(GeventWatcherObject handle, int revents);
static void python_stop(GeventWatcherObject handle);
static void _gevent_noop(void* handle) {}
static void (*gevent_noop)(void* handle) = &_gevent_noop;
static void _gevent_generic_callback1_unused(uv_handle_t* watcher, int arg)
{
// Python code may set this to NULL or even change it
// out from under us, which would tend to break things.
GeventWatcherObject handle = watcher->data;
const int cb_result = python_callback(handle, arg);
switch(cb_result) {
case -1:
// in case of exception, call self.loop.handle_error;
// this function is also responsible for stopping the watcher
// and allowing memory to be freed
python_handle_error(handle, arg);
break;
case 1:
// Code to stop the event IF NEEDED. Note that if python_callback
// has disposed of the last reference to the handle,
// `watcher` could now be invalid/disposed memory!
if (!uv_is_active(watcher)) {
if (watcher->data != handle) {
if (watcher->data) {
// If Python set the data to NULL, then they
// expected to be stopped. That's fine.
// Otherwise, something weird happened.
fprintf(stderr,
"WARNING: gevent: watcher handle changed in callback "
"from %p to %p for watcher at %p of type %d\n",
handle, watcher->data, watcher, watcher->type);
// There's a very good chance that the object the
// handle referred to has been changed and/or the
// old handle has been deallocated (most common), so
// passing the old handle will crash. Instead we
// pass a sigil to let python distinguish this case.
python_stop(NULL);
}
}
else {
python_stop(handle);
}
}
break;
case 2:
// watcher is already stopped and dead, nothing to do.
break;
default:
fprintf(stderr,
"WARNING: gevent: Unexpected return value %d from Python callback "
"for watcher %p (of type %d) and handle %p\n",
cb_result,
watcher, watcher->type, handle);
// XXX: Possible leaking of resources here? Should we be
// closing the watcher?
}
}
static void _gevent_generic_callback1(uv_handle_t* watcher, int arg)
{
python_queue_callback(watcher, arg);
}
static void _gevent_generic_callback0(uv_handle_t* handle)
{
_gevent_generic_callback1(handle, 0);
}
static void _gevent_async_callback0(uv_async_t* handle)
{
_gevent_generic_callback0((uv_handle_t*)handle);
}
static void _gevent_timer_callback0(uv_timer_t* handle)
{
_gevent_generic_callback0((uv_handle_t*)handle);
}
static void _gevent_prepare_callback0(uv_prepare_t* handle)
{
_gevent_generic_callback0((uv_handle_t*)handle);
}
static void _gevent_check_callback0(uv_check_t* handle)
{
_gevent_generic_callback0((uv_handle_t*)handle);
}
static void _gevent_idle_callback0(uv_idle_t* handle)
{
_gevent_generic_callback0((uv_handle_t*)handle);
}
static void _gevent_signal_callback1(uv_signal_t* handle, int signum)
{
_gevent_generic_callback1((uv_handle_t*)handle, signum);
}
static void _gevent_poll_callback2(void* handle, int status, int events)
{
_gevent_generic_callback1(handle, status < 0 ? status : events);
}
static void _gevent_fs_event_callback3(void* handle, const char* filename, int events, int status)
{
_gevent_generic_callback1(handle, status < 0 ? status : events);
}
typedef struct _gevent_fs_poll_s {
uv_fs_poll_t handle;
uv_stat_t curr;
uv_stat_t prev;
} gevent_fs_poll_t;
static void _gevent_fs_poll_callback3(void* handlep, int status, const uv_stat_t* prev, const uv_stat_t* curr)
{
// stat pointers are valid for this callback only.
// if given, copy them into our structure, where they can be reached
// from python, just like libev's watcher does, before calling
// the callback.
// The callback is invoked with status < 0 if path does not exist
// or is inaccessible. The watcher is not stopped but your
// callback is not called again until something changes (e.g. when
// the file is created or the error reason changes).
// In that case the fields will be 0 in curr/prev.
gevent_fs_poll_t* handle = (gevent_fs_poll_t*)handlep;
assert(status == 0);
handle->curr = *curr;
handle->prev = *prev;
_gevent_generic_callback1((uv_handle_t*)handle, 0);
}
static void gevent_uv_walk_callback_close(uv_handle_t* handle, void* arg)
{
if( handle && !uv_is_closing(handle) ) {
uv_close(handle, NULL);
}
}
static void gevent_close_all_handles(uv_loop_t* loop)
{
uv_walk(loop, gevent_uv_walk_callback_close, NULL);
}
static void gevent_zero_timer(uv_timer_t* handle)
{
memset(handle, 0, sizeof(uv_timer_t));
}
static void gevent_zero_check(uv_check_t* handle)
{
memset(handle, 0, sizeof(uv_check_t));
}
static void gevent_zero_prepare(uv_prepare_t* handle)
{
memset(handle, 0, sizeof(uv_prepare_t));
}
static void gevent_zero_loop(uv_loop_t* handle)
{
memset(handle, 0, sizeof(uv_loop_t));
}

601
python/gevent/libuv/loop.py Normal file
View File

@@ -0,0 +1,601 @@
"""
libuv loop implementation
"""
# pylint: disable=no-member
from __future__ import absolute_import, print_function
import os
from collections import defaultdict
from collections import namedtuple
from operator import delitem
import signal
from gevent._ffi import _dbg # pylint: disable=unused-import
from gevent._ffi.loop import AbstractLoop
from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error
from gevent._ffi.loop import assign_standard_callbacks
from gevent._ffi.loop import AbstractCallbacks
from gevent._util import implementer
from gevent._interfaces import ILoop
ffi = _corecffi.ffi
libuv = _corecffi.lib
__all__ = [
]
class _Callbacks(AbstractCallbacks):
def _find_loop_from_c_watcher(self, watcher_ptr):
loop_handle = ffi.cast('uv_handle_t*', watcher_ptr).data
return self.from_handle(loop_handle) if loop_handle else None
def python_sigchld_callback(self, watcher_ptr, _signum):
self.from_handle(ffi.cast('uv_handle_t*', watcher_ptr).data)._sigchld_callback()
def python_timer0_callback(self, watcher_ptr):
return self.python_prepare_callback(watcher_ptr)
def python_queue_callback(self, watcher_ptr, revents):
watcher_handle = watcher_ptr.data
the_watcher = self.from_handle(watcher_handle)
the_watcher.loop._queue_callback(watcher_ptr, revents)
_callbacks = assign_standard_callbacks(
ffi, libuv, _Callbacks,
[('python_sigchld_callback', None),
('python_timer0_callback', None),
('python_queue_callback', None)])
from gevent._ffi.loop import EVENTS
GEVENT_CORE_EVENTS = EVENTS # export
from gevent.libuv import watcher as _watchers # pylint:disable=no-name-in-module
_events_to_str = _watchers._events_to_str # export
READ = libuv.UV_READABLE
WRITE = libuv.UV_WRITABLE
def get_version():
uv_bytes = ffi.string(libuv.uv_version_string())
if not isinstance(uv_bytes, str):
# Py3
uv_str = uv_bytes.decode("ascii")
else:
uv_str = uv_bytes
return 'libuv-' + uv_str
def get_header_version():
return 'libuv-%d.%d.%d' % (libuv.UV_VERSION_MAJOR, libuv.UV_VERSION_MINOR, libuv.UV_VERSION_PATCH)
def supported_backends():
return ['default']
@implementer(ILoop)
class loop(AbstractLoop):
# XXX: Undocumented. Maybe better named 'timer_resolution'? We can't
# know this in general on libev
min_sleep_time = 0.001 # 1ms
error_handler = None
_CHECK_POINTER = 'uv_check_t *'
_PREPARE_POINTER = 'uv_prepare_t *'
_PREPARE_CALLBACK_SIG = "void(*)(void*)"
_TIMER_POINTER = _CHECK_POINTER # This is poorly named. It's for the callback "timer"
def __init__(self, flags=None, default=None):
AbstractLoop.__init__(self, ffi, libuv, _watchers, flags, default)
self.__loop_pid = os.getpid()
self._child_watchers = defaultdict(list)
self._io_watchers = dict()
self._fork_watchers = set()
self._pid = os.getpid()
self._default = self._ptr == libuv.uv_default_loop()
self._queued_callbacks = []
def _queue_callback(self, watcher_ptr, revents):
self._queued_callbacks.append((watcher_ptr, revents))
def _init_loop(self, flags, default):
if default is None:
default = True
# Unlike libev, libuv creates a new default
# loop automatically if the old default loop was
# closed.
if default:
# XXX: If the default loop had been destroyed, this
# will create a new one, but we won't destroy it
ptr = libuv.uv_default_loop()
else:
ptr = libuv.uv_loop_new()
if not ptr:
raise SystemError("Failed to get loop")
# Track whether or not any object has destroyed
# this loop. See _can_destroy_default_loop
ptr.data = ptr
return ptr
_signal_idle = None
def _init_and_start_check(self):
libuv.uv_check_init(self._ptr, self._check)
libuv.uv_check_start(self._check, libuv.python_check_callback)
libuv.uv_unref(self._check)
# We also have to have an idle watcher to be able to handle
# signals in a timely manner. Without them, libuv won't loop again
# and call into its check and prepare handlers.
# Note that this basically forces us into a busy-loop
# XXX: As predicted, using an idle watcher causes our process
# to eat 100% CPU time. We instead use a timer with a max of a .3 second
# delay to notice signals. Note that this timeout also implements fork
# watchers, effectively.
# XXX: Perhaps we could optimize this to notice when there are other
# timers in the loop and start/stop it then. When we have a callback
# scheduled, this should also be the same and unnecessary?
# libev does takes this basic approach on Windows.
self._signal_idle = ffi.new("uv_timer_t*")
libuv.uv_timer_init(self._ptr, self._signal_idle)
self._signal_idle.data = self._handle_to_self
sig_cb = ffi.cast('void(*)(uv_timer_t*)', libuv.python_check_callback)
libuv.uv_timer_start(self._signal_idle,
sig_cb,
300,
300)
libuv.uv_unref(self._signal_idle)
def _run_callbacks(self):
# Manually handle fork watchers.
curpid = os.getpid()
if curpid != self._pid:
self._pid = curpid
for watcher in self._fork_watchers:
watcher._on_fork()
# The contents of queued_callbacks at this point should be timers
# that expired when the loop began along with any idle watchers.
# We need to run them so that any manual callbacks they want to schedule
# get added to the list and ran next before we go on to poll for IO.
# This is critical for libuv on linux: closing a socket schedules some manual
# callbacks to actually stop the watcher; if those don't run before
# we poll for IO, then libuv can abort the process for the closed file descriptor.
# XXX: There's still a race condition here because we may not run *all* the manual
# callbacks. We need a way to prioritize those.
# Running these before the manual callbacks lead to some
# random test failures. In test__event.TestEvent_SetThenClear
# we would get a LoopExit sometimes. The problem occurred when
# a timer expired on entering the first loop; we would process
# it there, and then process the callback that it created
# below, leaving nothing for the loop to do. Having the
# self.run() manually process manual callbacks before
# continuing solves the problem. (But we must still run callbacks
# here again.)
self._prepare_ran_callbacks = self.__run_queued_callbacks()
super(loop, self)._run_callbacks()
def _init_and_start_prepare(self):
libuv.uv_prepare_init(self._ptr, self._prepare)
libuv.uv_prepare_start(self._prepare, libuv.python_prepare_callback)
libuv.uv_unref(self._prepare)
def _init_callback_timer(self):
libuv.uv_check_init(self._ptr, self._timer0)
def _stop_callback_timer(self):
libuv.uv_check_stop(self._timer0)
def _start_callback_timer(self):
# The purpose of the callback timer is to ensure that we run
# callbacks as soon as possible on the next iteration of the event loop.
# In libev, we set a 0 duration timer with a no-op callback.
# This executes immediately *after* the IO poll is done (it
# actually determines the time that the IO poll will block
# for), so having the timer present simply spins the loop, and
# our normal prepare watcher kicks in to run the callbacks.
# In libuv, however, timers are run *first*, before prepare
# callbacks and before polling for IO. So a no-op 0 duration
# timer actually does *nothing*. (Also note that libev queues all
# watchers found during IO poll to run at the end (I think), while libuv
# runs them in uv__io_poll itself.)
# From the loop inside uv_run:
# while True:
# uv__update_time(loop);
# uv__run_timers(loop);
# # we don't use pending watchers. They are how libuv
# # implements the pipe/udp/tcp streams.
# ran_pending = uv__run_pending(loop);
# uv__run_idle(loop);
# uv__run_prepare(loop);
# ...
# uv__io_poll(loop, timeout); # <--- IO watchers run here!
# uv__run_check(loop);
# libev looks something like this (pseudo code because the real code is
# hard to read):
#
# do {
# run_fork_callbacks();
# run_prepare_callbacks();
# timeout = min(time of all timers or normal block time)
# io_poll() # <--- Only queues IO callbacks
# update_now(); calculate_expired_timers();
# run callbacks in this order: (although specificying priorities changes it)
# check
# stat
# child
# signal
# timer
# io
# }
# So instead of running a no-op and letting the side-effect of spinning
# the loop run the callbacks, we must explicitly run them here.
# If we don't, test__systemerror:TestCallback will be flaky, failing
# one time out of ~20, depending on timing.
# To get them to run immediately after this current loop,
# we use a check watcher, instead of a 0 duration timer entirely.
# If we use a 0 duration timer, we can get stuck in a timer loop.
# Python 3.6 fails in test_ftplib.py
# As a final note, if we have not yet entered the loop *at
# all*, and a timer was created with a duration shorter than
# the amount of time it took for us to enter the loop in the
# first place, it may expire and get called before our callback
# does. This could also lead to test__systemerror:TestCallback
# appearing to be flaky.
# As yet another final note, if we are currently running a
# timer callback, meaning we're inside uv__run_timers() in C,
# and the Python starts a new timer, if the Python code then
# update's the loop's time, it's possible that timer will
# expire *and be run in the same iteration of the loop*. This
# is trivial to do: In sequential code, anything after
# `gevent.sleep(0.1)` is running in a timer callback. Starting
# a new timer---e.g., another gevent.sleep() call---will
# update the time, *before* uv__run_timers exits, meaning
# other timers get a chance to run before our check or prepare
# watcher callbacks do. Therefore, we do indeed have to have a 0
# timer to run callbacks---it gets inserted before any other user
# timers---ideally, this should be especially careful about how much time
# it runs for.
# AND YET: We can't actually do that. We get timeouts that I haven't fully
# investigated if we do. Probably stuck in a timer loop.
# As a partial remedy to this, unlike libev, our timer watcher
# class doesn't update the loop time by default.
libuv.uv_check_start(self._timer0, libuv.python_timer0_callback)
def _stop_aux_watchers(self):
assert self._prepare
assert self._check
assert self._signal_idle
libuv.uv_prepare_stop(self._prepare)
libuv.uv_ref(self._prepare) # Why are we doing this?
libuv.uv_check_stop(self._check)
libuv.uv_ref(self._check)
libuv.uv_timer_stop(self._signal_idle)
libuv.uv_ref(self._signal_idle)
libuv.uv_check_stop(self._timer0)
def _setup_for_run_callback(self):
self._start_callback_timer()
libuv.uv_ref(self._timer0)
def _can_destroy_loop(self, ptr):
# We're being asked to destroy a loop that's,
# at the time it was constructed, was the default loop.
# If loop objects were constructed more than once,
# it may have already been destroyed, though.
# We track this in the data member.
return ptr.data
def _destroy_loop(self, ptr):
ptr.data = ffi.NULL
libuv.uv_stop(ptr)
libuv.gevent_close_all_handles(ptr)
closed_failed = libuv.uv_loop_close(ptr)
if closed_failed:
assert closed_failed == libuv.UV_EBUSY
# We already closed all the handles. Run the loop
# once to let them be cut off from the loop.
ran_has_more_callbacks = libuv.uv_run(ptr, libuv.UV_RUN_ONCE)
if ran_has_more_callbacks:
libuv.uv_run(ptr, libuv.UV_RUN_NOWAIT)
closed_failed = libuv.uv_loop_close(ptr)
assert closed_failed == 0, closed_failed
# Destroy the native resources *after* we have closed
# the loop. If we do it before, walking the handles
# attached to the loop is likely to segfault.
libuv.gevent_zero_check(self._check)
libuv.gevent_zero_check(self._timer0)
libuv.gevent_zero_prepare(self._prepare)
libuv.gevent_zero_timer(self._signal_idle)
del self._check
del self._prepare
del self._signal_idle
del self._timer0
libuv.gevent_zero_loop(ptr)
# Destroy any watchers we're still holding on to.
del self._io_watchers
del self._fork_watchers
del self._child_watchers
def debug(self):
"""
Return all the handles that are open and their ref status.
"""
handle_state = namedtuple("HandleState",
['handle',
'type',
'watcher',
'ref',
'active',
'closing'])
handles = []
# XXX: Convert this to a modern callback.
def walk(handle, _arg):
data = handle.data
if data:
watcher = ffi.from_handle(data)
else:
watcher = None
handles.append(handle_state(handle,
ffi.string(libuv.uv_handle_type_name(handle.type)),
watcher,
libuv.uv_has_ref(handle),
libuv.uv_is_active(handle),
libuv.uv_is_closing(handle)))
libuv.uv_walk(self._ptr,
ffi.callback("void(*)(uv_handle_t*,void*)",
walk),
ffi.NULL)
return handles
def ref(self):
pass
def unref(self):
# XXX: Called by _run_callbacks.
pass
def break_(self, how=None):
libuv.uv_stop(self._ptr)
def reinit(self):
# TODO: How to implement? We probably have to simply
# re-__init__ this whole class? Does it matter?
# OR maybe we need to uv_walk() and close all the handles?
# XXX: libuv < 1.12 simply CANNOT handle a fork unless you immediately
# exec() in the child. There are multiple calls to abort() that
# will kill the child process:
# - The OS X poll implementation (kqueue) aborts on an error return
# value; since kqueue FDs can't be inherited, then the next call
# to kqueue in the child will fail and get aborted; fork() is likely
# to be called during the gevent loop, meaning we're deep inside the
# runloop already, so we can't even close the loop that we're in:
# it's too late, the next call to kqueue is already scheduled.
# - The threadpool, should it be in use, also aborts
# (https://github.com/joyent/libuv/pull/1136)
# - There global shared state that breaks signal handling
# and leads to an abort() in the child, EVEN IF the loop in the parent
# had already been closed
# (https://github.com/joyent/libuv/issues/1405)
# In 1.12, the uv_loop_fork function was added (by gevent!)
libuv.uv_loop_fork(self._ptr)
_prepare_ran_callbacks = False
def __run_queued_callbacks(self):
if not self._queued_callbacks:
return False
cbs = list(self._queued_callbacks)
self._queued_callbacks = []
for watcher_ptr, arg in cbs:
handle = watcher_ptr.data
if not handle:
# It's been stopped and possibly closed
assert not libuv.uv_is_active(watcher_ptr)
continue
val = _callbacks.python_callback(handle, arg)
if val == -1:
_callbacks.python_handle_error(handle, arg)
elif val == 1:
if not libuv.uv_is_active(watcher_ptr):
if watcher_ptr.data != handle:
if watcher_ptr.data:
_callbacks.python_stop(None)
else:
_callbacks.python_stop(handle)
return True
def run(self, nowait=False, once=False):
# we can only respect one flag or the other.
# nowait takes precedence because it can't block
mode = libuv.UV_RUN_DEFAULT
if once:
mode = libuv.UV_RUN_ONCE
if nowait:
mode = libuv.UV_RUN_NOWAIT
if mode == libuv.UV_RUN_DEFAULT:
while self._ptr and self._ptr.data:
# This is here to better preserve order guarantees. See _run_callbacks
# for details.
# It may get run again from the prepare watcher, so potentially we
# could take twice as long as the switch interval.
self._run_callbacks()
self._prepare_ran_callbacks = False
ran_status = libuv.uv_run(self._ptr, libuv.UV_RUN_ONCE)
# Note that we run queued callbacks when the prepare watcher runs,
# thus accounting for timers that expired before polling for IO,
# and idle watchers. This next call should get IO callbacks and
# callbacks from timers that expired *after* polling for IO.
ran_callbacks = self.__run_queued_callbacks()
if not ran_status and not ran_callbacks and not self._prepare_ran_callbacks:
# A return of 0 means there are no referenced and
# active handles. The loop is over.
# If we didn't run any callbacks, then we couldn't schedule
# anything to switch in the future, so there's no point
# running again.
return ran_status
return 0 # Somebody closed the loop
result = libuv.uv_run(self._ptr, mode)
self.__run_queued_callbacks()
return result
def now(self):
# libuv's now is expressed as an integer number of
# milliseconds, so to get it compatible with time.time units
# that this method is supposed to return, we have to divide by 1000.0
now = libuv.uv_now(self._ptr)
return now / 1000.0
def update_now(self):
libuv.uv_update_time(self._ptr)
def fileno(self):
if self._ptr:
fd = libuv.uv_backend_fd(self._ptr)
if fd >= 0:
return fd
_sigchld_watcher = None
_sigchld_callback_ffi = None
def install_sigchld(self):
if not self.default:
return
if self._sigchld_watcher:
return
self._sigchld_watcher = ffi.new('uv_signal_t*')
libuv.uv_signal_init(self._ptr, self._sigchld_watcher)
self._sigchld_watcher.data = self._handle_to_self
libuv.uv_signal_start(self._sigchld_watcher,
libuv.python_sigchld_callback,
signal.SIGCHLD)
def reset_sigchld(self):
if not self.default or not self._sigchld_watcher:
return
libuv.uv_signal_stop(self._sigchld_watcher)
# Must go through this to manage the memory lifetime
# correctly. Alternately, we could just stop it and restart
# it in install_sigchld?
_watchers.watcher._watcher_ffi_close(self._sigchld_watcher)
del self._sigchld_watcher
def _sigchld_callback(self):
# Signals can arrive at (relatively) any time. To eliminate
# race conditions, and behave more like libev, we "queue"
# sigchld to run when we run callbacks.
while True:
try:
pid, status, _usage = os.wait3(os.WNOHANG)
except OSError:
# Python 3 raises ChildProcessError
break
if pid == 0:
break
children_watchers = self._child_watchers.get(pid, []) + self._child_watchers.get(0, [])
for watcher in children_watchers:
self.run_callback(watcher._set_waitpid_status, pid, status)
# Don't invoke child watchers for 0 more than once
self._child_watchers[0] = []
def _register_child_watcher(self, watcher):
self._child_watchers[watcher._pid].append(watcher)
def _unregister_child_watcher(self, watcher):
try:
# stop() should be idempotent
self._child_watchers[watcher._pid].remove(watcher)
except ValueError:
pass
# Now's a good time to clean up any dead lists we don't need
# anymore
for pid in list(self._child_watchers):
if not self._child_watchers[pid]:
del self._child_watchers[pid]
def io(self, fd, events, ref=True, priority=None):
# We rely on hard references here and explicit calls to
# close() on the returned object to correctly manage
# the watcher lifetimes.
io_watchers = self._io_watchers
try:
io_watcher = io_watchers[fd]
assert io_watcher._multiplex_watchers, ("IO Watcher %s unclosed but should be dead" % io_watcher)
except KeyError:
# Start the watcher with just the events that we're interested in.
# as multiplexers are added, the real event mask will be updated to keep in sync.
# If we watch for too much, we get spurious wakeups and busy loops.
io_watcher = self._watchers.io(self, fd, 0)
io_watchers[fd] = io_watcher
io_watcher._no_more_watchers = lambda: delitem(io_watchers, fd)
return io_watcher.multiplex(events)
def prepare(self, ref=True, priority=None):
# We run arbitrary code in python_prepare_callback. That could switch
# greenlets. If it does that while also manipulating the active prepare
# watchers, we could corrupt the process state, since the prepare watcher
# queue is iterated on the stack (on unix). We could workaround this by implementing
# prepare watchers in pure Python.
# See https://github.com/gevent/gevent/issues/1126
raise TypeError("prepare watchers are not currently supported in libuv. "
"If you need them, please contact the maintainers.")

732
python/gevent/libuv/watcher.py Normal file
View File

@@ -0,0 +1,732 @@
# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable
# pylint: disable=no-member
from __future__ import absolute_import, print_function
import functools
import sys
from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error
ffi = _corecffi.ffi
libuv = _corecffi.lib
from gevent._ffi import watcher as _base
from gevent._ffi import _dbg
_closing_watchers = set()
# In debug mode, it would be nice to be able to clear the memory of
# the watcher (its size determined by
# libuv.uv_handle_size(ffi_watcher.type)) using memset so that if we
# are using it after it's supposedly been closed and deleted, we'd
# catch it sooner. BUT doing so breaks test__threadpool. We get errors
# about `pthread_mutex_lock[3]: Invalid argument` (and sometimes we
# crash) suggesting either that we're writing on memory that doesn't
# belong to us, somehow, or that we haven't actually lost all
# references...
_uv_close_callback = ffi.def_extern(name='_uv_close_callback')(_closing_watchers.remove)
_events = [(libuv.UV_READABLE, "READ"),
(libuv.UV_WRITABLE, "WRITE")]
def _events_to_str(events): # export
return _base.events_to_str(events, _events)
class UVFuncallError(ValueError):
pass
class libuv_error_wrapper(object):
# Makes sure that everything stored as a function
# on the wrapper instances (classes, actually,
# because this is used by the metaclass)
# checks its return value and raises an error.
# This expects that everything we call has an int
# or void return value and follows the conventions
# of error handling (that negative values are errors)
def __init__(self, uv):
self._libuv = uv
def __getattr__(self, name):
libuv_func = getattr(self._libuv, name)
@functools.wraps(libuv_func)
def wrap(*args, **kwargs):
if args and isinstance(args[0], watcher):
args = args[1:]
res = libuv_func(*args, **kwargs)
if res is not None and res < 0:
raise UVFuncallError(
str(ffi.string(libuv.uv_err_name(res)).decode('ascii')
+ ' '
+ ffi.string(libuv.uv_strerror(res)).decode('ascii'))
+ " Args: " + repr(args) + " KWARGS: " + repr(kwargs)
)
return res
setattr(self, name, wrap)
return wrap
class ffi_unwrapper(object):
# undoes the wrapping of libuv_error_wrapper for
# the methods used by the metaclass that care
def __init__(self, ff):
self._ffi = ff
def __getattr__(self, name):
return getattr(self._ffi, name)
def addressof(self, lib, name):
assert isinstance(lib, libuv_error_wrapper)
return self._ffi.addressof(libuv, name)
class watcher(_base.watcher):
_FFI = ffi_unwrapper(ffi)
_LIB = libuv_error_wrapper(libuv)
_watcher_prefix = 'uv'
_watcher_struct_pattern = '%s_t'
@classmethod
def _watcher_ffi_close(cls, ffi_watcher):
# Managing the lifetime of _watcher is tricky.
# They have to be uv_close()'d, but that only
# queues them to be closed in the *next* loop iteration.
# The memory must stay valid for at least that long,
# or assert errors are triggered. We can't use a ffi.gc()
# pointer to queue the uv_close, because by the time the
# destructor is called, there's no way to keep the memory alive
# and it could be re-used.
# So here we resort to resurrecting the pointer object out
# of our scope, keeping it alive past this object's lifetime.
# We then use the uv_close callback to handle removing that
# reference. There's no context passed to the close callback,
# so we have to do this globally.
# Sadly, doing this causes crashes if there were multiple
# watchers for a given FD, so we have to take special care
# about that. See https://github.com/gevent/gevent/issues/790#issuecomment-208076604
# Note that this cannot be a __del__ method, because we store
# the CFFI handle to self on self, which is a cycle, and
# objects with a __del__ method cannot be collected on CPython < 3.4
# Instead, this is arranged as a callback to GC when the
# watcher class dies. Obviously it's important to keep the ffi
# watcher alive.
# We can pass in "subclasses" if uv_handle_t that line up at the C level,
# but that don't in CFFI without a cast. But be careful what we use the cast
# for, don't pass it back to C.
ffi_handle_watcher = cls._FFI.cast('uv_handle_t*', ffi_watcher)
if ffi_handle_watcher.type and not libuv.uv_is_closing(ffi_watcher):
# If the type isn't set, we were never properly initialized,
# and trying to close it results in libuv terminating the process.
# Sigh. Same thing if it's already in the process of being
# closed.
_closing_watchers.add(ffi_watcher)
libuv.uv_close(ffi_watcher, libuv._uv_close_callback)
ffi_handle_watcher.data = ffi.NULL
def _watcher_ffi_set_init_ref(self, ref):
self.ref = ref
def _watcher_ffi_init(self, args):
# TODO: we could do a better job chokepointing this
return self._watcher_init(self.loop.ptr,
self._watcher,
*args)
def _watcher_ffi_start(self):
self._watcher_start(self._watcher, self._watcher_callback)
def _watcher_ffi_stop(self):
if self._watcher:
# The multiplexed io watcher deletes self._watcher
# when it closes down. If that's in the process of
# an error handler, AbstractCallbacks.unhandled_onerror
# will try to close us again.
self._watcher_stop(self._watcher)
@_base.only_if_watcher
def _watcher_ffi_ref(self):
libuv.uv_ref(self._watcher)
@_base.only_if_watcher
def _watcher_ffi_unref(self):
libuv.uv_unref(self._watcher)
def _watcher_ffi_start_unref(self):
pass
def _watcher_ffi_stop_ref(self):
pass
def _get_ref(self):
# Convert 1/0 to True/False
if self._watcher is None:
return None
return True if libuv.uv_has_ref(self._watcher) else False
def _set_ref(self, value):
if value:
self._watcher_ffi_ref()
else:
self._watcher_ffi_unref()
ref = property(_get_ref, _set_ref)
def feed(self, _revents, _callback, *_args):
raise Exception("Not implemented")
class io(_base.IoMixin, watcher):
_watcher_type = 'poll'
_watcher_callback_name = '_gevent_poll_callback2'
# On Windows is critical to be able to garbage collect these
# objects in a timely fashion so that they don't get reused
# for multiplexing completely different sockets. This is because
# uv_poll_init_socket does a lot of setup for the socket to make
# polling work. If get reused for another socket that has the same
# fileno, things break badly. (In theory this could be a problem
# on posix too, but in practice it isn't).
# TODO: We should probably generalize this to all
# ffi watchers. Avoiding GC cycles as much as possible
# is a good thing, and potentially allocating new handles
# as needed gets us better memory locality.
# Especially on Windows, we must also account for the case that a
# reference to this object has leaked (e.g., the socket object is
# still around), but the fileno has been closed and a new one
# opened. We must still get a new native watcher at that point. We
# handle this case by simply making sure that we don't even have
# a native watcher until the object is started, and we shut it down
# when the object is stopped.
# XXX: I was able to solve at least Windows test_ftplib.py issues
# with more of a careful use of io objects in socket.py, so
# delaying this entirely is at least temporarily on hold. Instead
# sticking with the _watcher_create function override for the
# moment.
# XXX: Note 2: Moving to a deterministic close model, which was necessary
# for PyPy, also seems to solve the Windows issues. So we're completely taking
# this object out of the loop's registration; we don't want GC callbacks and
# uv_close anywhere *near* this object.
_watcher_registers_with_loop_on_create = False
EVENT_MASK = libuv.UV_READABLE | libuv.UV_WRITABLE | libuv.UV_DISCONNECT
_multiplex_watchers = ()
def __init__(self, loop, fd, events, ref=True, priority=None):
super(io, self).__init__(loop, fd, events, ref=ref, priority=priority, _args=(fd,))
self._fd = fd
self._events = events
self._multiplex_watchers = []
def _get_fd(self):
return self._fd
@_base.not_while_active
def _set_fd(self, fd):
self._fd = fd
self._watcher_ffi_init((fd,))
def _get_events(self):
return self._events
def _set_events(self, events):
if events == self._events:
return
self._events = events
if self.active:
# We're running but libuv specifically says we can
# call start again to change our event mask.
assert self._handle is not None
self._watcher_start(self._watcher, self._events, self._watcher_callback)
events = property(_get_events, _set_events)
def _watcher_ffi_start(self):
self._watcher_start(self._watcher, self._events, self._watcher_callback)
if sys.platform.startswith('win32'):
# uv_poll can only handle sockets on Windows, but the plain
# uv_poll_init we call on POSIX assumes that the fileno
# argument is already a C fileno, as created by
# _get_osfhandle. C filenos are limited resources, must be
# closed with _close. So there are lifetime issues with that:
# calling the C function _close to dispose of the fileno
# *also* closes the underlying win32 handle, possibly
# prematurely. (XXX: Maybe could do something with weak
# references? But to what?)
# All libuv wants to do with the fileno in uv_poll_init is
# turn it back into a Win32 SOCKET handle.
# Now, libuv provides uv_poll_init_socket, which instead of
# taking a C fileno takes the SOCKET, avoiding the need to dance with
# the C runtime.
# It turns out that SOCKET (win32 handles in general) can be
# represented with `intptr_t`. It further turns out that
# CPython *directly* exposes the SOCKET handle as the value of
# fileno (32-bit PyPy does some munging on it, which should
# rarely matter). So we can pass socket.fileno() through
# to uv_poll_init_socket.
# See _corecffi_build.
_watcher_init = watcher._LIB.uv_poll_init_socket
class _multiplexwatcher(object):
callback = None
args = ()
pass_events = False
ref = True
def __init__(self, events, watcher):
self._events = events
# References:
# These objects must keep the original IO object alive;
# the IO object SHOULD NOT keep these alive to avoid cycles
# We MUST NOT rely on GC to clean up the IO objects, but the explicit
# calls to close(); see _multiplex_closed.
self._watcher_ref = watcher
events = property(
lambda self: self._events,
_base.not_while_active(lambda self, nv: setattr(self, '_events', nv)))
def start(self, callback, *args, **kwargs):
self.pass_events = kwargs.get("pass_events")
self.callback = callback
self.args = args
watcher = self._watcher_ref
if watcher is not None:
if not watcher.active:
watcher._io_start()
else:
# Make sure we're in the event mask
watcher._calc_and_update_events()
def stop(self):
self.callback = None
self.pass_events = None
self.args = None
watcher = self._watcher_ref
if watcher is not None:
watcher._io_maybe_stop()
def close(self):
if self._watcher_ref is not None:
self._watcher_ref._multiplex_closed(self)
self._watcher_ref = None
@property
def active(self):
return self.callback is not None
@property
def _watcher(self):
# For testing.
return self._watcher_ref._watcher
# ares.pyx depends on this property,
# and test__core uses it too
fd = property(lambda self: getattr(self._watcher_ref, '_fd', -1),
lambda self, nv: self._watcher_ref._set_fd(nv))
def _io_maybe_stop(self):
self._calc_and_update_events()
for w in self._multiplex_watchers:
if w.callback is not None:
# There's still a reference to it, and it's started,
# so we can't stop.
return
# If we get here, nothing was started
# so we can take ourself out of the polling set
self.stop()
def _io_start(self):
self._calc_and_update_events()
self.start(self._io_callback, pass_events=True)
def _calc_and_update_events(self):
events = 0
for watcher in self._multiplex_watchers:
if watcher.callback is not None:
# Only ask for events that are active.
events |= watcher.events
self._set_events(events)
def multiplex(self, events):
watcher = self._multiplexwatcher(events, self)
self._multiplex_watchers.append(watcher)
self._calc_and_update_events()
return watcher
def close(self):
super(io, self).close()
del self._multiplex_watchers
def _multiplex_closed(self, watcher):
self._multiplex_watchers.remove(watcher)
if not self._multiplex_watchers:
self.stop() # should already be stopped
self._no_more_watchers()
# It is absolutely critical that we control when the call
# to uv_close() gets made. uv_close() of a uv_poll_t
# handle winds up calling uv__platform_invalidate_fd,
# which, as the name implies, destroys any outstanding
# events for the *fd* that haven't been delivered yet, and also removes
# the *fd* from the poll set. So if this happens later, at some
# non-deterministic time when (cyclic or otherwise) GC runs,
# *and* we've opened a new watcher for the fd, that watcher will
# suddenly and mysteriously stop seeing events. So we do this now;
# this method is smart enough not to close the handle twice.
self.close()
else:
self._calc_and_update_events()
def _no_more_watchers(self):
# The loop sets this on an individual watcher to delete it from
# the active list where it keeps hard references.
pass
def _io_callback(self, events):
if events < 0:
# actually a status error code
_dbg("Callback error on", self._fd,
ffi.string(libuv.uv_err_name(events)),
ffi.string(libuv.uv_strerror(events)))
# XXX: We've seen one half of a FileObjectPosix pair
# (the read side of a pipe) report errno 11 'bad file descriptor'
# after the write side was closed and its watcher removed. But
# we still need to attempt to read from it to clear out what's in
# its buffers--if we return with the watcher inactive before proceeding to wake up
# the reader, we get a LoopExit. So we can't return here and arguably shouldn't print it
# either. The negative events mask will match the watcher's mask.
# See test__fileobject.py:Test.test_newlines for an example.
# On Windows (at least with PyPy), we can get ENOTSOCK (socket operation on non-socket)
# if a socket gets closed. If we don't pass the events on, we hang.
# See test__makefile_ref.TestSSL for examples.
# return
for watcher in self._multiplex_watchers:
if not watcher.callback:
# Stopped
continue
assert watcher._watcher_ref is self, (self, watcher._watcher_ref)
send_event = (events & watcher.events) or events < 0
if send_event:
if not watcher.pass_events:
watcher.callback(*watcher.args)
else:
watcher.callback(events, *watcher.args)
class _SimulatedWithAsyncMixin(object):
_watcher_skip_ffi = True
def __init__(self, loop, *args, **kwargs):
self._async = loop.async_()
try:
super(_SimulatedWithAsyncMixin, self).__init__(loop, *args, **kwargs)
except:
self._async.close()
raise
def _watcher_create(self, _args):
return
@property
def _watcher_handle(self):
return None
def _watcher_ffi_init(self, _args):
return
def _watcher_ffi_set_init_ref(self, ref):
self._async.ref = ref
@property
def active(self):
return self._async.active
def start(self, cb, *args):
self._register_loop_callback()
self.callback = cb
self.args = args
self._async.start(cb, *args)
#watcher.start(self, cb, *args)
def stop(self):
self._unregister_loop_callback()
self.callback = None
self.args = None
self._async.stop()
def close(self):
if self._async is not None:
a = self._async
#self._async = None
a.close()
def _register_loop_callback(self):
# called from start()
raise NotImplementedError()
def _unregister_loop_callback(self):
# called from stop
raise NotImplementedError()
class fork(_SimulatedWithAsyncMixin,
_base.ForkMixin,
watcher):
# We'll have to implement this one completely manually
# Right now it doesn't matter much since libuv doesn't survive
# a fork anyway. (That's a work in progress)
_watcher_skip_ffi = False
def _register_loop_callback(self):
self.loop._fork_watchers.add(self)
def _unregister_loop_callback(self):
try:
# stop() should be idempotent
self.loop._fork_watchers.remove(self)
except KeyError:
pass
def _on_fork(self):
self._async.send()
class child(_SimulatedWithAsyncMixin,
_base.ChildMixin,
watcher):
_watcher_skip_ffi = True
# We'll have to implement this one completely manually.
# Our approach is to use a SIGCHLD handler and the original
# os.waitpid call.
# On Unix, libuv's uv_process_t and uv_spawn use SIGCHLD,
# just like libev does for its child watchers. So
# we're not adding any new SIGCHLD related issues not already
# present in libev.
def _register_loop_callback(self):
self.loop._register_child_watcher(self)
def _unregister_loop_callback(self):
self.loop._unregister_child_watcher(self)
def _set_waitpid_status(self, pid, status):
self._rpid = pid
self._rstatus = status
self._async.send()
class async_(_base.AsyncMixin, watcher):
_watcher_callback_name = '_gevent_async_callback0'
def _watcher_ffi_init(self, args):
# It's dangerous to have a raw, non-initted struct
# around; it will crash in uv_close() when we get GC'd,
# and send() will also crash.
# NOTE: uv_async_init is NOT idempotent. Calling it more than
# once adds the uv_async_t to the internal queue multiple times,
# and uv_close only cleans up one of them, meaning that we tend to
# crash. Thus we have to be very careful not to allow that.
return self._watcher_init(self.loop.ptr, self._watcher, ffi.NULL)
def _watcher_ffi_start(self):
# we're created in a started state, but we didn't provide a
# callback (because if we did and we don't have a value in our
# callback attribute, then python_callback would crash.) Note that
# uv_async_t->async_cb is not technically documented as public.
self._watcher.async_cb = self._watcher_callback
def _watcher_ffi_stop(self):
self._watcher.async_cb = ffi.NULL
# We have to unref this because we're setting the cb behind libuv's
# back, basically: once a async watcher is started, it can't ever be
# stopped through libuv interfaces, so it would never lose its active
# status, and thus if it stays reffed it would keep the event loop
# from exiting.
self._watcher_ffi_unref()
def send(self):
if libuv.uv_is_closing(self._watcher):
raise Exception("Closing handle")
libuv.uv_async_send(self._watcher)
@property
def pending(self):
return None
locals()['async'] = async_
class timer(_base.TimerMixin, watcher):
_watcher_callback_name = '_gevent_timer_callback0'
# In libuv, timer callbacks continue running while any timer is
# expired, including newly added timers. Newly added non-zero
# timers (especially of small duration) can be seen to be expired
# if the loop time is updated while we are in a timer callback.
# This can lead to us being stuck running timers for a terribly
# long time, which is not good. So default to not updating the
# time.
# Also, newly-added timers of 0 duration can *also* stall the
# loop, because they'll be seen to be expired immediately.
# Updating the time can prevent that, *if* there was already a
# timer for a longer duration scheduled.
# To mitigate the above problems, our loop implementation turns
# zero duration timers into check watchers instead using OneShotCheck.
# This ensures the loop cycles. Of course, the 'again' method does
# nothing on them and doesn't exist. In practice that's not an issue.
_again = False
def _watcher_ffi_init(self, args):
self._watcher_init(self.loop._ptr, self._watcher)
self._after, self._repeat = args
if self._after and self._after < 0.001:
import warnings
# XXX: The stack level is hard to determine, could be getting here
# through a number of different ways.
warnings.warn("libuv only supports millisecond timer resolution; "
"all times less will be set to 1 ms",
stacklevel=6)
# The alternative is to effectively pass in int(0.1) == 0, which
# means no sleep at all, which leads to excessive wakeups
self._after = 0.001
if self._repeat and self._repeat < 0.001:
import warnings
warnings.warn("libuv only supports millisecond timer resolution; "
"all times less will be set to 1 ms",
stacklevel=6)
self._repeat = 0.001
def _watcher_ffi_start(self):
if self._again:
libuv.uv_timer_again(self._watcher)
else:
try:
self._watcher_start(self._watcher, self._watcher_callback,
int(self._after * 1000),
int(self._repeat * 1000))
except ValueError:
# in case of non-ints in _after/_repeat
raise TypeError()
def again(self, callback, *args, **kw):
if not self.active:
# If we've never been started, this is the same as starting us.
# libuv makes the distinction, libev doesn't.
self.start(callback, *args, **kw)
return
self._again = True
try:
self.start(callback, *args, **kw)
finally:
del self._again
class stat(_base.StatMixin, watcher):
_watcher_type = 'fs_poll'
_watcher_struct_name = 'gevent_fs_poll_t'
_watcher_callback_name = '_gevent_fs_poll_callback3'
def _watcher_set_data(self, the_watcher, data):
the_watcher.handle.data = data
return data
def _watcher_ffi_init(self, args):
return self._watcher_init(self.loop._ptr, self._watcher)
MIN_STAT_INTERVAL = 0.1074891 # match libev; 0.0 is default
def _watcher_ffi_start(self):
# libev changes this when the watcher is started
if self._interval < self.MIN_STAT_INTERVAL:
self._interval = self.MIN_STAT_INTERVAL
self._watcher_start(self._watcher, self._watcher_callback,
self._cpath,
int(self._interval * 1000))
@property
def _watcher_handle(self):
return self._watcher.handle.data
@property
def attr(self):
if not self._watcher.curr.st_nlink:
return
return self._watcher.curr
@property
def prev(self):
if not self._watcher.prev.st_nlink:
return
return self._watcher.prev
class signal(_base.SignalMixin, watcher):
_watcher_callback_name = '_gevent_signal_callback1'
def _watcher_ffi_init(self, args):
self._watcher_init(self.loop._ptr, self._watcher)
self.ref = False # libev doesn't ref these by default
def _watcher_ffi_start(self):
self._watcher_start(self._watcher, self._watcher_callback,
self._signalnum)
class idle(_base.IdleMixin, watcher):
# Because libuv doesn't support priorities, idle watchers are
# potentially quite a bit different than under libev
_watcher_callback_name = '_gevent_idle_callback0'
class check(_base.CheckMixin, watcher):
_watcher_callback_name = '_gevent_check_callback0'
class OneShotCheck(check):
_watcher_skip_ffi = True
def __make_cb(self, func):
stop = self.stop
@functools.wraps(func)
def cb(*args):
stop()
return func(*args)
return cb
def start(self, callback, *args):
return check.start(self, self.__make_cb(callback), *args)
class prepare(_base.PrepareMixin, watcher):
_watcher_callback_name = '_gevent_prepare_callback0'