Windows: Use 32-bit distribution of python

python/gevent/_monitor.py

@@ -0,0 +1,325 @@
+# Copyright (c) 2018 gevent. See LICENSE for details.
+from __future__ import print_function, absolute_import, division
+
+import os
+import sys
+
+from weakref import ref as wref
+
+from greenlet import getcurrent
+
+from gevent import config as GEVENT_CONFIG
+from gevent.monkey import get_original
+from gevent.events import notify
+from gevent.events import EventLoopBlocked
+from gevent.events import MemoryUsageThresholdExceeded
+from gevent.events import MemoryUsageUnderThreshold
+from gevent.events import IPeriodicMonitorThread
+from gevent.events import implementer
+
+from gevent._tracer import GreenletTracer
+from gevent._compat import thread_mod_name
+from gevent._compat import perf_counter
+
+
+
+__all__ = [
+    'PeriodicMonitoringThread',
+]
+
+get_thread_ident = get_original(thread_mod_name, 'get_ident')
+start_new_thread = get_original(thread_mod_name, 'start_new_thread')
+thread_sleep = get_original('time', 'sleep')
+
+
+
+class MonitorWarning(RuntimeWarning):
+    """The type of warnings we emit."""
+
+
+class _MonitorEntry(object):
+
+    __slots__ = ('function', 'period', 'last_run_time')
+
+    def __init__(self, function, period):
+        self.function = function
+        self.period = period
+        self.last_run_time = 0
+
+    def __eq__(self, other):
+        return self.function == other.function and self.period == other.period
+
+    def __repr__(self):
+        return repr((self.function, self.period, self.last_run_time))
+
+
+@implementer(IPeriodicMonitorThread)
+class PeriodicMonitoringThread(object):
+    # This doesn't extend threading.Thread because that gets monkey-patched.
+    # We use the low-level 'start_new_thread' primitive instead.
+
+    # The amount of seconds we will sleep when we think we have nothing
+    # to do.
+    inactive_sleep_time = 2.0
+
+    # The absolute minimum we will sleep, regardless of
+    # what particular monitoring functions want to say.
+    min_sleep_time = 0.005
+
+    # The minimum period in seconds at which we will check memory usage.
+    # Getting memory usage is fairly expensive.
+    min_memory_monitor_period = 2
+
+    # A list of _MonitorEntry objects: [(function(hub), period, last_run_time))]
+    # The first entry is always our entry for self.monitor_blocking
+    _monitoring_functions = None
+
+    # The calculated min sleep time for the monitoring functions list.
+    _calculated_sleep_time = None
+
+    # A boolean value that also happens to capture the
+    # memory usage at the time we exceeded the threshold. Reset
+    # to 0 when we go back below.
+    _memory_exceeded = 0
+
+    # The instance of GreenletTracer we're using
+    _greenlet_tracer = None
+
+    def __init__(self, hub):
+        self._hub_wref = wref(hub, self._on_hub_gc)
+        self.should_run = True
+
+        # Must be installed in the thread that the hub is running in;
+        # the trace function is threadlocal
+        assert get_thread_ident() == hub.thread_ident
+        self._greenlet_tracer = GreenletTracer()
+
+        self._monitoring_functions = [_MonitorEntry(self.monitor_blocking,
+                                                    GEVENT_CONFIG.max_blocking_time)]
+        self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
+        # Create the actual monitoring thread. This is effectively a "daemon"
+        # thread.
+        self.monitor_thread_ident = start_new_thread(self, ())
+
+        # We must track the PID to know if your thread has died after a fork
+        self.pid = os.getpid()
+
+    def _on_fork(self):
+        # Pseudo-standard method that resolver_ares and threadpool
+        # also have, called by hub.reinit()
+        pid = os.getpid()
+        if pid != self.pid:
+            self.pid = pid
+            self.monitor_thread_ident = start_new_thread(self, ())
+
+    @property
+    def hub(self):
+        return self._hub_wref()
+
+
+    def monitoring_functions(self):
+        # Return a list of _MonitorEntry objects
+
+        # Update max_blocking_time each time.
+        mbt = GEVENT_CONFIG.max_blocking_time # XXX: Events so we know when this changes.
+        if mbt != self._monitoring_functions[0].period:
+            self._monitoring_functions[0].period = mbt
+            self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
+        return self._monitoring_functions
+
+    def add_monitoring_function(self, function, period):
+        if not callable(function):
+            raise ValueError("function must be callable")
+
+        if period is None:
+            # Remove.
+            self._monitoring_functions = [
+                x for x in self._monitoring_functions
+                if x.function != function
+            ]
+        elif period <= 0:
+            raise ValueError("Period must be positive.")
+        else:
+            # Add or update period
+            entry = _MonitorEntry(function, period)
+            self._monitoring_functions = [
+                x if x.function != function else entry
+                for x in self._monitoring_functions
+            ]
+            if entry not in self._monitoring_functions:
+                self._monitoring_functions.append(entry)
+        self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
+
+    def calculate_sleep_time(self):
+        min_sleep = self._calculated_sleep_time
+        if min_sleep <= 0:
+            # Everyone wants to be disabled. Sleep for a longer period of
+            # time than usual so we don't spin unnecessarily. We might be
+            # enabled again in the future.
+            return self.inactive_sleep_time
+        return max((min_sleep, self.min_sleep_time))
+
+    def kill(self):
+        if not self.should_run:
+            # Prevent overwriting trace functions.
+            return
+        # Stop this monitoring thread from running.
+        self.should_run = False
+        # Uninstall our tracing hook
+        self._greenlet_tracer.kill()
+
+    def _on_hub_gc(self, _):
+        self.kill()
+
+    def __call__(self):
+        # The function that runs in the monitoring thread.
+        # We cannot use threading.current_thread because it would
+        # create an immortal DummyThread object.
+        getcurrent().gevent_monitoring_thread = wref(self)
+
+        try:
+            while self.should_run:
+                functions = self.monitoring_functions()
+                assert functions
+                sleep_time = self.calculate_sleep_time()
+
+                thread_sleep(sleep_time)
+
+                # Make sure the hub is still around, and still active,
+                # and keep it around while we are here.
+                hub = self.hub
+                if not hub:
+                    self.kill()
+
+                if self.should_run:
+                    this_run = perf_counter()
+                    for entry in functions:
+                        f = entry.function
+                        period = entry.period
+                        last_run = entry.last_run_time
+                        if period and last_run + period <= this_run:
+                            entry.last_run_time = this_run
+                            f(hub)
+                del hub # break our reference to hub while we sleep
+
+        except SystemExit:
+            pass
+        except: # pylint:disable=bare-except
+            # We're a daemon thread, so swallow any exceptions that get here
+            # during interpreter shutdown.
+            if not sys or not sys.stderr: # pragma: no cover
+                # Interpreter is shutting down
+                pass
+            else:
+                hub = self.hub
+                if hub is not None:
+                    # XXX: This tends to do bad things like end the process, because we
+                    # try to switch *threads*, which can't happen. Need something better.
+                    hub.handle_error(self, *sys.exc_info())
+
+    def monitor_blocking(self, hub):
+        # Called periodically to see if the trace function has
+        # fired to switch greenlets. If not, we will print
+        # the greenlet tree.
+
+        # For tests, we return a true value when we think we found something
+        # blocking
+
+        did_block = self._greenlet_tracer.did_block_hub(hub)
+        if not did_block:
+            return
+
+        active_greenlet = did_block[1]
+        report = self._greenlet_tracer.did_block_hub_report(
+            hub, active_greenlet,
+            dict(greenlet_stacks=False, current_thread_ident=self.monitor_thread_ident))
+
+        stream = hub.exception_stream
+        for line in report:
+            # Printing line by line may interleave with other things,
+            # but it should also prevent a "reentrant call to print"
+            # when the report is large.
+            print(line, file=stream)
+
+        notify(EventLoopBlocked(active_greenlet, GEVENT_CONFIG.max_blocking_time, report))
+        return (active_greenlet, report)
+
+    def ignore_current_greenlet_blocking(self):
+        self._greenlet_tracer.ignore_current_greenlet_blocking()
+
+    def monitor_current_greenlet_blocking(self):
+        self._greenlet_tracer.monitor_current_greenlet_blocking()
+
+    def _get_process(self): # pylint:disable=method-hidden
+        try:
+            # The standard library 'resource' module doesn't provide
+            # a standard way to get the RSS measure, only the maximum.
+            # You might be tempted to try to compute something by adding
+            # together text and data sizes, but on many systems those come back
+            # zero. So our only option is psutil.
+            from psutil import Process, AccessDenied
+            # Make sure it works (why would we be denied access to our own process?)
+            try:
+                proc = Process()
+                proc.memory_full_info()
+            except AccessDenied: # pragma: no cover
+                proc = None
+        except ImportError:
+            proc = None
+
+        self._get_process = lambda: proc
+        return proc
+
+    def can_monitor_memory_usage(self):
+        return self._get_process() is not None
+
+    def install_monitor_memory_usage(self):
+        # Start monitoring memory usage, if possible.
+        # If not possible, emit a warning.
+        if not self.can_monitor_memory_usage():
+            import warnings
+            warnings.warn("Unable to monitor memory usage. Install psutil.",
+                          MonitorWarning)
+            return
+
+        self.add_monitoring_function(self.monitor_memory_usage,
+                                     max(GEVENT_CONFIG.memory_monitor_period,
+                                         self.min_memory_monitor_period))
+
+    def monitor_memory_usage(self, _hub):
+        max_allowed = GEVENT_CONFIG.max_memory_usage
+        if not max_allowed:
+            # They disabled it.
+            return -1 # value for tests
+
+        rusage = self._get_process().memory_full_info()
+        # uss only documented available on Windows, Linux, and OS X.
+        # If not available, fall back to rss as an aproximation.
+        mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
+
+        event = None # Return value for tests
+
+        if mem_usage > max_allowed:
+            if mem_usage > self._memory_exceeded:
+                # We're still growing
+                event = MemoryUsageThresholdExceeded(
+                    mem_usage, max_allowed, rusage)
+                notify(event)
+            self._memory_exceeded = mem_usage
+        else:
+            # we're below. Were we above it last time?
+            if self._memory_exceeded:
+                event = MemoryUsageUnderThreshold(
+                    mem_usage, max_allowed, rusage, self._memory_exceeded)
+                notify(event)
+            self._memory_exceeded = 0
+
+        return event
+
+    def __repr__(self):
+        return '<%s at %s in thread %s greenlet %r for %r>' % (
+            self.__class__.__name__,
+            hex(id(self)),
+            hex(self.monitor_thread_ident),
+            getcurrent(),
+            self._hub_wref())