heckyel/yt-local
0
0
Fork 0
Code Issues Pull Requests Releases Activity

Windows: Use 32-bit distribution of python

Browse Source
This commit is contained in:
James Taylor
2018-09-14 19:32:27 -07:00
parent 6ca20ff701
commit 4212164e91
166 changed files with 175548 additions and 44620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

536
python/gevent/util.py
View File

@@ -3,12 +3,39 @@
Low-level utilities.
"""
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import functools
import gc
import pprint
import sys
import traceback
__all__ = ['wrap_errors']
from greenlet import getcurrent
from greenlet import greenlet as RawGreenlet
from gevent._compat import PYPY
from gevent._compat import thread_mod_name
from gevent._util import _NONE
__all__ = [
'format_run_info',
'print_run_info',
'GreenletTree',
'wrap_errors',
'assert_switches',
]
# PyPy is very slow at formatting stacks
# for some reason.
_STACK_LIMIT = 20 if PYPY else None
def _noop():
return None
def _ready():
return False
class wrap_errors(object):
"""
@@ -58,3 +85,508 @@ class wrap_errors(object):
def __getattr__(self, name):
return getattr(self.__func, name)
def print_run_info(thread_stacks=True, greenlet_stacks=True, limit=_NONE, file=None):
"""
Call `format_run_info` and print the results to *file*.
If *file* is not given, `sys.stderr` will be used.
.. versionadded:: 1.3b1
"""
lines = format_run_info(thread_stacks=thread_stacks,
greenlet_stacks=greenlet_stacks,
limit=limit)
file = sys.stderr if file is None else file
for l in lines:
print(l, file=file)
def format_run_info(thread_stacks=True,
greenlet_stacks=True,
limit=_NONE,
current_thread_ident=None):
"""
format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]
Request information about the running threads of the current process.
This is a debugging utility. Its output has no guarantees other than being
intended for human consumption.
:keyword bool thread_stacks: If true, then include the stacks for
running threads.
:keyword bool greenlet_stacks: If true, then include the stacks for
running greenlets. (Spawning stacks will always be printed.)
Setting this to False can reduce the output volume considerably
without reducing the overall information if *thread_stacks* is true
and you can associate a greenlet to a thread (using ``thread_ident``
printed values).
:keyword int limit: If given, passed directly to `traceback.format_stack`.
If not given, this defaults to the whole stack under CPython, and a
smaller stack under PyPy.
:return: A sequence of text lines detailing the stacks of running
threads and greenlets. (One greenlet will duplicate one thread,
the current thread and greenlet. If there are multiple running threads,
the stack for the current greenlet may be incorrectly duplicated in multiple
greenlets.)
Extra information about
:class:`gevent.Greenlet` object will also be returned.
.. versionadded:: 1.3a1
.. versionchanged:: 1.3a2
Renamed from ``dump_stacks`` to reflect the fact that this
prints additional information about greenlets, including their
spawning stack, parent, locals, and any spawn tree locals.
.. versionchanged:: 1.3b1
Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
"""
if current_thread_ident is None:
from gevent import monkey
current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')()
lines = []
limit = _STACK_LIMIT if limit is _NONE else limit
_format_thread_info(lines, thread_stacks, limit, current_thread_ident)
_format_greenlet_info(lines, greenlet_stacks, limit)
return lines
def _format_thread_info(lines, thread_stacks, limit, current_thread_ident):
import threading
threads = {th.ident: th for th in threading.enumerate()}
lines.append('*' * 80)
lines.append('* Threads')
thread = None
frame = None
for thread_ident, frame in sys._current_frames().items():
lines.append("*" * 80)
thread = threads.get(thread_ident)
name = thread.name if thread else None
if getattr(thread, 'gevent_monitoring_thread', None):
name = repr(thread.gevent_monitoring_thread())
if current_thread_ident == thread_ident:
name = '%s) (CURRENT' % (name,)
lines.append('Thread 0x%x (%s)\n' % (thread_ident, name))
if thread_stacks:
lines.append(''.join(traceback.format_stack(frame, limit)))
else:
lines.append('\t...stack elided...')
# We may have captured our own frame, creating a reference
# cycle, so clear it out.
del thread
del frame
del lines
del threads
def _format_greenlet_info(lines, greenlet_stacks, limit):
# Use the gc module to inspect all objects to find the greenlets
# since there isn't a global registry
lines.append('*' * 80)
lines.append('* Greenlets')
lines.append('*' * 80)
for tree in GreenletTree.forest():
lines.extend(tree.format_lines(details={
'running_stacks': greenlet_stacks,
'running_stack_limit': limit,
}))
del lines
dump_stacks = format_run_info
def _line(f):
@functools.wraps(f)
def w(self, *args, **kwargs):
r = f(self, *args, **kwargs)
self.lines.append(r)
return w
class _TreeFormatter(object):
UP_AND_RIGHT = '+'
HORIZONTAL = '-'
VERTICAL = '|'
VERTICAL_AND_RIGHT = '+'
DATA = ':'
label_space = 1
horiz_width = 3
indent = 1
def __init__(self, details, depth=0):
self.lines = []
self.depth = depth
self.details = details
if not details:
self.child_data = lambda *args, **kwargs: None
def deeper(self):
return type(self)(self.details, self.depth + 1)
@_line
def node_label(self, text):
return text
@_line
def child_head(self, label, right=VERTICAL_AND_RIGHT):
return (
' ' * self.indent
+ right
+ self.HORIZONTAL * self.horiz_width
+ ' ' * self.label_space
+ label
)
def last_child_head(self, label):
return self.child_head(label, self.UP_AND_RIGHT)
@_line
def child_tail(self, line, vertical=VERTICAL):
return (
' ' * self.indent
+ vertical
+ ' ' * self.horiz_width
+ line
)
def last_child_tail(self, line):
return self.child_tail(line, vertical=' ' * len(self.VERTICAL))
@_line
def child_data(self, data, data_marker=DATA): # pylint:disable=method-hidden
return ((
' ' * self.indent
+ (data_marker if not self.depth else ' ')
+ ' ' * self.horiz_width
+ ' ' * self.label_space
+ data
),)
def last_child_data(self, data):
return self.child_data(data, ' ')
def child_multidata(self, data):
# Remove embedded newlines
for l in data.splitlines():
self.child_data(l)
class GreenletTree(object):
"""
Represents a tree of greenlets.
In gevent, the *parent* of a greenlet is usually the hub, so this
tree is primarily arganized along the *spawning_greenlet* dimension.
This object has a small str form showing this hierarchy. The `format`
method can output more details. The exact output is unspecified but is
intended to be human readable.
Use the `forest` method to get the root greenlet trees for
all threads, and the `current_tree` to get the root greenlet tree for
the current thread.
"""
#: The greenlet this tree represents.
greenlet = None
def __init__(self, greenlet):
self.greenlet = greenlet
self.child_trees = []
def add_child(self, tree):
if tree is self:
return
self.child_trees.append(tree)
@property
def root(self):
return self.greenlet.parent is None
def __getattr__(self, name):
return getattr(self.greenlet, name)
DEFAULT_DETAILS = {
'running_stacks': True,
'running_stack_limit': _STACK_LIMIT,
'spawning_stacks': True,
'locals': True,
}
def format_lines(self, details=True):
"""
Return a sequence of lines for the greenlet tree.
:keyword bool details: If true (the default),
then include more informative details in the output.
"""
if not isinstance(details, dict):
if not details:
details = {}
else:
details = self.DEFAULT_DETAILS.copy()
else:
params = details
details = self.DEFAULT_DETAILS.copy()
details.update(params)
tree = _TreeFormatter(details, depth=0)
lines = [l[0] if isinstance(l, tuple) else l
for l in self._render(tree)]
return lines
def format(self, details=True):
"""
Like `format_lines` but returns a string.
"""
lines = self.format_lines(details)
return '\n'.join(lines)
def __str__(self):
return self.format(False)
@staticmethod
def __render_tb(tree, label, frame, limit):
tree.child_data(label)
tb = ''.join(traceback.format_stack(frame, limit))
tree.child_multidata(tb)
@staticmethod
def __spawning_parent(greenlet):
return (getattr(greenlet, 'spawning_greenlet', None) or _noop)()
def __render_locals(self, tree):
# Defer the import to avoid cycles
from gevent.local import all_local_dicts_for_greenlet
gr_locals = all_local_dicts_for_greenlet(self.greenlet)
if gr_locals:
tree.child_data("Greenlet Locals:")
for (kind, idl), vals in gr_locals:
tree.child_data(" Local %s at %s" % (kind, hex(idl)))
tree.child_multidata(" " + pprint.pformat(vals))
def _render(self, tree):
label = repr(self.greenlet)
if not self.greenlet: # Not running or dead
# raw greenlets do not have ready
if getattr(self.greenlet, 'ready', _ready)():
label += '; finished'
if self.greenlet.value is not None:
label += ' with value ' + repr(self.greenlet.value)[:30]
elif getattr(self.greenlet, 'exception', None) is not None:
label += ' with exception ' + repr(self.greenlet.exception)
else:
label += '; not running'
tree.node_label(label)
tree.child_data('Parent: ' + repr(self.greenlet.parent))
if getattr(self.greenlet, 'gevent_monitoring_thread', None) is not None:
tree.child_data('Monitoring Thread:' + repr(self.greenlet.gevent_monitoring_thread()))
if self.greenlet and tree.details and tree.details['running_stacks']:
self.__render_tb(tree, 'Running:', self.greenlet.gr_frame,
tree.details['running_stack_limit'])
spawning_stack = getattr(self.greenlet, 'spawning_stack', None)
if spawning_stack and tree.details and tree.details['spawning_stacks']:
# We already placed a limit on the spawning stack when we captured it.
self.__render_tb(tree, 'Spawned at:', spawning_stack, None)
spawning_parent = self.__spawning_parent(self.greenlet)
tree_locals = getattr(self.greenlet, 'spawn_tree_locals', None)
if tree_locals and tree_locals is not getattr(spawning_parent, 'spawn_tree_locals', None):
tree.child_data('Spawn Tree Locals')
tree.child_multidata(pprint.pformat(tree_locals))
self.__render_locals(tree)
self.__render_children(tree)
return tree.lines
def __render_children(self, tree):
children = sorted(self.child_trees,
key=lambda c: (
# raw greenlets first
getattr(c, 'minimal_ident', -1),
# running greenlets first
getattr(c, 'ready', _ready)(),
id(c.parent)))
for n, child in enumerate(children):
child_tree = child._render(tree.deeper())
head = tree.child_head
tail = tree.child_tail
data = tree.child_data
if n == len(children) - 1:
# last child does not get the line drawn
head = tree.last_child_head
tail = tree.last_child_tail
data = tree.last_child_data
head(child_tree.pop(0))
for child_data in child_tree:
if isinstance(child_data, tuple):
data(child_data[0])
else:
tail(child_data)
return tree.lines
@staticmethod
def _root_greenlet(greenlet):
while greenlet.parent is not None and not getattr(greenlet, 'greenlet_tree_is_root', False):
greenlet = greenlet.parent
return greenlet
@classmethod
def _forest(cls):
main_greenlet = cls._root_greenlet(getcurrent())
trees = {}
roots = {}
current_tree = roots[main_greenlet] = trees[main_greenlet] = cls(main_greenlet)
for ob in gc.get_objects():
if not isinstance(ob, RawGreenlet):
continue
if getattr(ob, 'greenlet_tree_is_ignored', False):
continue
spawn_parent = cls.__spawning_parent(ob)
if spawn_parent is None:
root = cls._root_greenlet(ob)
try:
tree = roots[root]
except KeyError: # pragma: no cover
tree = GreenletTree(root)
roots[root] = trees[root] = tree
else:
try:
tree = trees[spawn_parent]
except KeyError: # pragma: no cover
tree = trees[spawn_parent] = cls(spawn_parent)
try:
child_tree = trees[ob]
except KeyError:
trees[ob] = child_tree = cls(ob)
tree.add_child(child_tree)
return roots, current_tree
@classmethod
def forest(cls):
"""
forest() -> sequence
Return a sequence of `GreenletTree`, one for each running
native thread.
"""
return list(cls._forest()[0].values())
@classmethod
def current_tree(cls):
"""
current_tree() -> GreenletTree
Returns the `GreenletTree` for the current thread.
"""
return cls._forest()[1]
class _FailedToSwitch(AssertionError):
pass
class assert_switches(object):
"""
A context manager for ensuring a block of code switches greenlets.
This performs a similar function as the :doc:`monitoring thread
</monitoring>`, but the scope is limited to the body of the with
statement. If the code within the body doesn't yield to the hub
(and doesn't raise an exception), then upon exiting the
context manager an :exc:`AssertionError` will be raised.
This is useful in unit tests and for debugging purposes.
:keyword float max_blocking_time: If given, the body is allowed
to block for up to this many fractional seconds before
an error is raised.
:keyword bool hub_only: If True, then *max_blocking_time* only
refers to the amount of time spent between switches into the
hub. If False, then it refers to the maximum time between
*any* switches. If *max_blocking_time* is not given, has no
effect.
Example::
# This will always raise an exception: nothing switched
with assert_switches():
pass
# This will never raise an exception; nothing switched,
# but it happened very fast
with assert_switches(max_blocking_time=1.0):
pass
.. versionadded:: 1.3
"""
hub = None
tracer = None
def __init__(self, max_blocking_time=None, hub_only=False):
self.max_blocking_time = max_blocking_time
self.hub_only = hub_only
def __enter__(self):
from gevent import get_hub
from gevent import _tracer
self.hub = hub = get_hub()
# TODO: We could optimize this to use the GreenletTracer
# installed by the monitoring thread, if there is one.
# As it is, we will chain trace calls back to it.
if not self.max_blocking_time:
self.tracer = _tracer.GreenletTracer()
elif self.hub_only:
self.tracer = _tracer.HubSwitchTracer(hub, self.max_blocking_time)
else:
self.tracer = _tracer.MaxSwitchTracer(hub, self.max_blocking_time)
self.tracer.monitor_current_greenlet_blocking()
return self
def __exit__(self, t, v, tb):
self.tracer.kill()
hub = self.hub; self.hub = None
tracer = self.tracer; self.tracer = None
# Only check if there was no exception raised, we
# don't want to hide anything
if t is not None:
return
did_block = tracer.did_block_hub(hub)
if did_block:
active_greenlet = did_block[1]
report_lines = tracer.did_block_hub_report(hub, active_greenlet, {})
raise _FailedToSwitch('\n'.join(report_lines))