Merge flask framework and other stuff from master

http_errors.py

@@ -1,16 +0,0 @@
-class Code2xx(Exception):
-    pass
-class Code200(Code2xx):
-    pass
-
-class Error4xx(Exception):
-    pass
-class Error404(Error4xx):
-    pass
-
-class Error5xx(Exception):
-    pass
-class Error500(Error5xx):
-    pass
-class Error502(Error5xx):
-    pass

python/click/__init__.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+"""
+click
+~~~~~
+
+Click is a simple Python module inspired by the stdlib optparse to make
+writing command line scripts fun. Unlike other modules, it's based
+around a simple API that does not come with too much magic and is
+composable.
+
+:copyright: © 2014 by the Pallets team.
+:license: BSD, see LICENSE.rst for more details.
+"""
+
+# Core classes
+from .core import Context, BaseCommand, Command, MultiCommand, Group, \
+     CommandCollection, Parameter, Option, Argument
+
+# Globals
+from .globals import get_current_context
+
+# Decorators
+from .decorators import pass_context, pass_obj, make_pass_decorator, \
+     command, group, argument, option, confirmation_option, \
+     password_option, version_option, help_option
+
+# Types
+from .types import ParamType, File, Path, Choice, IntRange, Tuple, \
+     DateTime, STRING, INT, FLOAT, BOOL, UUID, UNPROCESSED, FloatRange
+
+# Utilities
+from .utils import echo, get_binary_stream, get_text_stream, open_file, \
+     format_filename, get_app_dir, get_os_args
+
+# Terminal functions
+from .termui import prompt, confirm, get_terminal_size, echo_via_pager, \
+     progressbar, clear, style, unstyle, secho, edit, launch, getchar, \
+     pause
+
+# Exceptions
+from .exceptions import ClickException, UsageError, BadParameter, \
+     FileError, Abort, NoSuchOption, BadOptionUsage, BadArgumentUsage, \
+     MissingParameter
+
+# Formatting
+from .formatting import HelpFormatter, wrap_text
+
+# Parsing
+from .parser import OptionParser
+
+
+__all__ = [
+    # Core classes
+    'Context', 'BaseCommand', 'Command', 'MultiCommand', 'Group',
+    'CommandCollection', 'Parameter', 'Option', 'Argument',
+
+    # Globals
+    'get_current_context',
+
+    # Decorators
+    'pass_context', 'pass_obj', 'make_pass_decorator', 'command', 'group',
+    'argument', 'option', 'confirmation_option', 'password_option',
+    'version_option', 'help_option',
+
+    # Types
+    'ParamType', 'File', 'Path', 'Choice', 'IntRange', 'Tuple',
+    'DateTime', 'STRING', 'INT', 'FLOAT', 'BOOL', 'UUID', 'UNPROCESSED',
+    'FloatRange',
+
+    # Utilities
+    'echo', 'get_binary_stream', 'get_text_stream', 'open_file',
+    'format_filename', 'get_app_dir', 'get_os_args',
+
+    # Terminal functions
+    'prompt', 'confirm', 'get_terminal_size', 'echo_via_pager',
+    'progressbar', 'clear', 'style', 'unstyle', 'secho', 'edit', 'launch',
+    'getchar', 'pause',
+
+    # Exceptions
+    'ClickException', 'UsageError', 'BadParameter', 'FileError',
+    'Abort', 'NoSuchOption', 'BadOptionUsage', 'BadArgumentUsage',
+    'MissingParameter',
+
+    # Formatting
+    'HelpFormatter', 'wrap_text',
+
+    # Parsing
+    'OptionParser',
+]
+
+
+# Controls if click should emit the warning about the use of unicode
+# literals.
+disable_unicode_literals_warning = False
+
+
+__version__ = '7.0'

python/click/_bashcomplete.py

@@ -0,0 +1,293 @@
+import copy
+import os
+import re
+
+from .utils import echo
+from .parser import split_arg_string
+from .core import MultiCommand, Option, Argument
+from .types import Choice
+
+try:
+    from collections import abc
+except ImportError:
+    import collections as abc
+
+WORDBREAK = '='
+
+# Note, only BASH version 4.4 and later have the nosort option.
+COMPLETION_SCRIPT_BASH = '''
+%(complete_func)s() {
+    local IFS=$'\n'
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\
+                   COMP_CWORD=$COMP_CWORD \\
+                   %(autocomplete_var)s=complete $1 ) )
+    return 0
+}
+
+%(complete_func)setup() {
+    local COMPLETION_OPTIONS=""
+    local BASH_VERSION_ARR=(${BASH_VERSION//./ })
+    # Only BASH version 4.4 and later have the nosort option.
+    if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] && [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then
+        COMPLETION_OPTIONS="-o nosort"
+    fi
+
+    complete $COMPLETION_OPTIONS -F %(complete_func)s %(script_names)s
+}
+
+%(complete_func)setup
+'''
+
+COMPLETION_SCRIPT_ZSH = '''
+%(complete_func)s() {
+    local -a completions
+    local -a completions_with_descriptions
+    local -a response
+    response=("${(@f)$( env COMP_WORDS=\"${words[*]}\" \\
+                        COMP_CWORD=$((CURRENT-1)) \\
+                        %(autocomplete_var)s=\"complete_zsh\" \\
+                        %(script_names)s )}")
+
+    for key descr in ${(kv)response}; do
+      if [[ "$descr" == "_" ]]; then
+          completions+=("$key")
+      else
+          completions_with_descriptions+=("$key":"$descr")
+      fi
+    done
+
+    if [ -n "$completions_with_descriptions" ]; then
+        _describe -V unsorted completions_with_descriptions -U -Q
+    fi
+
+    if [ -n "$completions" ]; then
+        compadd -U -V unsorted -Q -a completions
+    fi
+    compstate[insert]="automenu"
+}
+
+compdef %(complete_func)s %(script_names)s
+'''
+
+_invalid_ident_char_re = re.compile(r'[^a-zA-Z0-9_]')
+
+
+def get_completion_script(prog_name, complete_var, shell):
+    cf_name = _invalid_ident_char_re.sub('', prog_name.replace('-', '_'))
+    script = COMPLETION_SCRIPT_ZSH if shell == 'zsh' else COMPLETION_SCRIPT_BASH
+    return (script % {
+        'complete_func': '_%s_completion' % cf_name,
+        'script_names': prog_name,
+        'autocomplete_var': complete_var,
+    }).strip() + ';'
+
+
+def resolve_ctx(cli, prog_name, args):
+    """
+    Parse into a hierarchy of contexts. Contexts are connected through the parent variable.
+    :param cli: command definition
+    :param prog_name: the program that is running
+    :param args: full list of args
+    :return: the final context/command parsed
+    """
+    ctx = cli.make_context(prog_name, args, resilient_parsing=True)
+    args = ctx.protected_args + ctx.args
+    while args:
+        if isinstance(ctx.command, MultiCommand):
+            if not ctx.command.chain:
+                cmd_name, cmd, args = ctx.command.resolve_command(ctx, args)
+                if cmd is None:
+                    return ctx
+                ctx = cmd.make_context(cmd_name, args, parent=ctx,
+                                       resilient_parsing=True)
+                args = ctx.protected_args + ctx.args
+            else:
+                # Walk chained subcommand contexts saving the last one.
+                while args:
+                    cmd_name, cmd, args = ctx.command.resolve_command(ctx, args)
+                    if cmd is None:
+                        return ctx
+                    sub_ctx = cmd.make_context(cmd_name, args, parent=ctx,
+                                               allow_extra_args=True,
+                                               allow_interspersed_args=False,
+                                               resilient_parsing=True)
+                    args = sub_ctx.args
+                ctx = sub_ctx
+                args = sub_ctx.protected_args + sub_ctx.args
+        else:
+            break
+    return ctx
+
+
+def start_of_option(param_str):
+    """
+    :param param_str: param_str to check
+    :return: whether or not this is the start of an option declaration (i.e. starts "-" or "--")
+    """
+    return param_str and param_str[:1] == '-'
+
+
+def is_incomplete_option(all_args, cmd_param):
+    """
+    :param all_args: the full original list of args supplied
+    :param cmd_param: the current command paramter
+    :return: whether or not the last option declaration (i.e. starts "-" or "--") is incomplete and
+    corresponds to this cmd_param. In other words whether this cmd_param option can still accept
+    values
+    """
+    if not isinstance(cmd_param, Option):
+        return False
+    if cmd_param.is_flag:
+        return False
+    last_option = None
+    for index, arg_str in enumerate(reversed([arg for arg in all_args if arg != WORDBREAK])):
+        if index + 1 > cmd_param.nargs:
+            break
+        if start_of_option(arg_str):
+            last_option = arg_str
+
+    return True if last_option and last_option in cmd_param.opts else False
+
+
+def is_incomplete_argument(current_params, cmd_param):
+    """
+    :param current_params: the current params and values for this argument as already entered
+    :param cmd_param: the current command parameter
+    :return: whether or not the last argument is incomplete and corresponds to this cmd_param. In
+    other words whether or not the this cmd_param argument can still accept values
+    """
+    if not isinstance(cmd_param, Argument):
+        return False
+    current_param_values = current_params[cmd_param.name]
+    if current_param_values is None:
+        return True
+    if cmd_param.nargs == -1:
+        return True
+    if isinstance(current_param_values, abc.Iterable) \
+            and cmd_param.nargs > 1 and len(current_param_values) < cmd_param.nargs:
+        return True
+    return False
+
+
+def get_user_autocompletions(ctx, args, incomplete, cmd_param):
+    """
+    :param ctx: context associated with the parsed command
+    :param args: full list of args
+    :param incomplete: the incomplete text to autocomplete
+    :param cmd_param: command definition
+    :return: all the possible user-specified completions for the param
+    """
+    results = []
+    if isinstance(cmd_param.type, Choice):
+        # Choices don't support descriptions.
+        results = [(c, None)
+                   for c in cmd_param.type.choices if str(c).startswith(incomplete)]
+    elif cmd_param.autocompletion is not None:
+        dynamic_completions = cmd_param.autocompletion(ctx=ctx,
+                                                       args=args,
+                                                       incomplete=incomplete)
+        results = [c if isinstance(c, tuple) else (c, None)
+                   for c in dynamic_completions]
+    return results
+
+
+def get_visible_commands_starting_with(ctx, starts_with):
+    """
+    :param ctx: context associated with the parsed command
+    :starts_with: string that visible commands must start with.
+    :return: all visible (not hidden) commands that start with starts_with.
+    """
+    for c in ctx.command.list_commands(ctx):
+        if c.startswith(starts_with):
+            command = ctx.command.get_command(ctx, c)
+            if not command.hidden:
+                yield command
+
+
+def add_subcommand_completions(ctx, incomplete, completions_out):
+    # Add subcommand completions.
+    if isinstance(ctx.command, MultiCommand):
+        completions_out.extend(
+            [(c.name, c.get_short_help_str()) for c in get_visible_commands_starting_with(ctx, incomplete)])
+
+    # Walk up the context list and add any other completion possibilities from chained commands
+    while ctx.parent is not None:
+        ctx = ctx.parent
+        if isinstance(ctx.command, MultiCommand) and ctx.command.chain:
+            remaining_commands = [c for c in get_visible_commands_starting_with(ctx, incomplete)
+                                  if c.name not in ctx.protected_args]
+            completions_out.extend([(c.name, c.get_short_help_str()) for c in remaining_commands])
+
+
+def get_choices(cli, prog_name, args, incomplete):
+    """
+    :param cli: command definition
+    :param prog_name: the program that is running
+    :param args: full list of args
+    :param incomplete: the incomplete text to autocomplete
+    :return: all the possible completions for the incomplete
+    """
+    all_args = copy.deepcopy(args)
+
+    ctx = resolve_ctx(cli, prog_name, args)
+    if ctx is None:
+        return []
+
+    # In newer versions of bash long opts with '='s are partitioned, but it's easier to parse
+    # without the '='
+    if start_of_option(incomplete) and WORDBREAK in incomplete:
+        partition_incomplete = incomplete.partition(WORDBREAK)
+        all_args.append(partition_incomplete[0])
+        incomplete = partition_incomplete[2]
+    elif incomplete == WORDBREAK:
+        incomplete = ''
+
+    completions = []
+    if start_of_option(incomplete):
+        # completions for partial options
+        for param in ctx.command.params:
+            if isinstance(param, Option) and not param.hidden:
+                param_opts = [param_opt for param_opt in param.opts +
+                              param.secondary_opts if param_opt not in all_args or param.multiple]
+                completions.extend([(o, param.help) for o in param_opts if o.startswith(incomplete)])
+        return completions
+    # completion for option values from user supplied values
+    for param in ctx.command.params:
+        if is_incomplete_option(all_args, param):
+            return get_user_autocompletions(ctx, all_args, incomplete, param)
+    # completion for argument values from user supplied values
+    for param in ctx.command.params:
+        if is_incomplete_argument(ctx.params, param):
+            return get_user_autocompletions(ctx, all_args, incomplete, param)
+
+    add_subcommand_completions(ctx, incomplete, completions)
+    # Sort before returning so that proper ordering can be enforced in custom types.
+    return sorted(completions)
+
+
+def do_complete(cli, prog_name, include_descriptions):
+    cwords = split_arg_string(os.environ['COMP_WORDS'])
+    cword = int(os.environ['COMP_CWORD'])
+    args = cwords[1:cword]
+    try:
+        incomplete = cwords[cword]
+    except IndexError:
+        incomplete = ''
+
+    for item in get_choices(cli, prog_name, args, incomplete):
+        echo(item[0])
+        if include_descriptions:
+            # ZSH has trouble dealing with empty array parameters when returned from commands, so use a well defined character '_' to indicate no description is present.
+            echo(item[1] if item[1] else '_')
+
+    return True
+
+
+def bashcomplete(cli, prog_name, complete_var, complete_instr):
+    if complete_instr.startswith('source'):
+        shell = 'zsh' if complete_instr == 'source_zsh' else 'bash'
+        echo(get_completion_script(prog_name, complete_var, shell))
+        return True
+    elif complete_instr == 'complete' or complete_instr == 'complete_zsh':
+        return do_complete(cli, prog_name, complete_instr == 'complete_zsh')
+    return False

python/click/_compat.py

@@ -0,0 +1,703 @@
+import re
+import io
+import os
+import sys
+import codecs
+from weakref import WeakKeyDictionary
+
+
+PY2 = sys.version_info[0] == 2
+CYGWIN = sys.platform.startswith('cygwin')
+# Determine local App Engine environment, per Google's own suggestion
+APP_ENGINE = ('APPENGINE_RUNTIME' in os.environ and
+              'Development/' in os.environ['SERVER_SOFTWARE'])
+WIN = sys.platform.startswith('win') and not APP_ENGINE
+DEFAULT_COLUMNS = 80
+
+
+_ansi_re = re.compile(r'\033\[((?:\d|;)*)([a-zA-Z])')
+
+
+def get_filesystem_encoding():
+    return sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
+def _make_text_stream(stream, encoding, errors,
+                      force_readable=False, force_writable=False):
+    if encoding is None:
+        encoding = get_best_encoding(stream)
+    if errors is None:
+        errors = 'replace'
+    return _NonClosingTextIOWrapper(stream, encoding, errors,
+                                    line_buffering=True,
+                                    force_readable=force_readable,
+                                    force_writable=force_writable)
+
+
+def is_ascii_encoding(encoding):
+    """Checks if a given encoding is ascii."""
+    try:
+        return codecs.lookup(encoding).name == 'ascii'
+    except LookupError:
+        return False
+
+
+def get_best_encoding(stream):
+    """Returns the default stream encoding if not found."""
+    rv = getattr(stream, 'encoding', None) or sys.getdefaultencoding()
+    if is_ascii_encoding(rv):
+        return 'utf-8'
+    return rv
+
+
+class _NonClosingTextIOWrapper(io.TextIOWrapper):
+
+    def __init__(self, stream, encoding, errors,
+                 force_readable=False, force_writable=False, **extra):
+        self._stream = stream = _FixupStream(stream, force_readable,
+                                             force_writable)
+        io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra)
+
+    # The io module is a place where the Python 3 text behavior
+    # was forced upon Python 2, so we need to unbreak
+    # it to look like Python 2.
+    if PY2:
+        def write(self, x):
+            if isinstance(x, str) or is_bytes(x):
+                try:
+                    self.flush()
+                except Exception:
+                    pass
+                return self.buffer.write(str(x))
+            return io.TextIOWrapper.write(self, x)
+
+        def writelines(self, lines):
+            for line in lines:
+                self.write(line)
+
+    def __del__(self):
+        try:
+            self.detach()
+        except Exception:
+            pass
+
+    def isatty(self):
+        # https://bitbucket.org/pypy/pypy/issue/1803
+        return self._stream.isatty()
+
+
+class _FixupStream(object):
+    """The new io interface needs more from streams than streams
+    traditionally implement.  As such, this fix-up code is necessary in
+    some circumstances.
+
+    The forcing of readable and writable flags are there because some tools
+    put badly patched objects on sys (one such offender are certain version
+    of jupyter notebook).
+    """
+
+    def __init__(self, stream, force_readable=False, force_writable=False):
+        self._stream = stream
+        self._force_readable = force_readable
+        self._force_writable = force_writable
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+    def read1(self, size):
+        f = getattr(self._stream, 'read1', None)
+        if f is not None:
+            return f(size)
+        # We only dispatch to readline instead of read in Python 2 as we
+        # do not want cause problems with the different implementation
+        # of line buffering.
+        if PY2:
+            return self._stream.readline(size)
+        return self._stream.read(size)
+
+    def readable(self):
+        if self._force_readable:
+            return True
+        x = getattr(self._stream, 'readable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.read(0)
+        except Exception:
+            return False
+        return True
+
+    def writable(self):
+        if self._force_writable:
+            return True
+        x = getattr(self._stream, 'writable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.write('')
+        except Exception:
+            try:
+                self._stream.write(b'')
+            except Exception:
+                return False
+        return True
+
+    def seekable(self):
+        x = getattr(self._stream, 'seekable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.seek(self._stream.tell())
+        except Exception:
+            return False
+        return True
+
+
+if PY2:
+    text_type = unicode
+    bytes = str
+    raw_input = raw_input
+    string_types = (str, unicode)
+    int_types = (int, long)
+    iteritems = lambda x: x.iteritems()
+    range_type = xrange
+
+    def is_bytes(x):
+        return isinstance(x, (buffer, bytearray))
+
+    _identifier_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
+
+    # For Windows, we need to force stdout/stdin/stderr to binary if it's
+    # fetched for that.  This obviously is not the most correct way to do
+    # it as it changes global state.  Unfortunately, there does not seem to
+    # be a clear better way to do it as just reopening the file in binary
+    # mode does not change anything.
+    #
+    # An option would be to do what Python 3 does and to open the file as
+    # binary only, patch it back to the system, and then use a wrapper
+    # stream that converts newlines.  It's not quite clear what's the
+    # correct option here.
+    #
+    # This code also lives in _winconsole for the fallback to the console
+    # emulation stream.
+    #
+    # There are also Windows environments where the `msvcrt` module is not
+    # available (which is why we use try-catch instead of the WIN variable
+    # here), such as the Google App Engine development server on Windows. In
+    # those cases there is just nothing we can do.
+    def set_binary_mode(f):
+        return f
+
+    try:
+        import msvcrt
+    except ImportError:
+        pass
+    else:
+        def set_binary_mode(f):
+            try:
+                fileno = f.fileno()
+            except Exception:
+                pass
+            else:
+                msvcrt.setmode(fileno, os.O_BINARY)
+            return f
+
+    try:
+        import fcntl
+    except ImportError:
+        pass
+    else:
+        def set_binary_mode(f):
+            try:
+                fileno = f.fileno()
+            except Exception:
+                pass
+            else:
+                flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+                fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)
+            return f
+
+    def isidentifier(x):
+        return _identifier_re.search(x) is not None
+
+    def get_binary_stdin():
+        return set_binary_mode(sys.stdin)
+
+    def get_binary_stdout():
+        _wrap_std_stream('stdout')
+        return set_binary_mode(sys.stdout)
+
+    def get_binary_stderr():
+        _wrap_std_stream('stderr')
+        return set_binary_mode(sys.stderr)
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdin, encoding, errors,
+                                 force_readable=True)
+
+    def get_text_stdout(encoding=None, errors=None):
+        _wrap_std_stream('stdout')
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdout, encoding, errors,
+                                 force_writable=True)
+
+    def get_text_stderr(encoding=None, errors=None):
+        _wrap_std_stream('stderr')
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stderr, encoding, errors,
+                                 force_writable=True)
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), 'replace')
+        return value
+else:
+    import io
+    text_type = str
+    raw_input = input
+    string_types = (str,)
+    int_types = (int,)
+    range_type = range
+    isidentifier = lambda x: x.isidentifier()
+    iteritems = lambda x: iter(x.items())
+
+    def is_bytes(x):
+        return isinstance(x, (bytes, memoryview, bytearray))
+
+    def _is_binary_reader(stream, default=False):
+        try:
+            return isinstance(stream.read(0), bytes)
+        except Exception:
+            return default
+            # This happens in some cases where the stream was already
+            # closed.  In this case, we assume the default.
+
+    def _is_binary_writer(stream, default=False):
+        try:
+            stream.write(b'')
+        except Exception:
+            try:
+                stream.write('')
+                return False
+            except Exception:
+                pass
+            return default
+        return True
+
+    def _find_binary_reader(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detaching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_reader(stream, False):
+            return stream
+
+        buf = getattr(stream, 'buffer', None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_reader(buf, True):
+            return buf
+
+    def _find_binary_writer(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detatching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_writer(stream, False):
+            return stream
+
+        buf = getattr(stream, 'buffer', None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_writer(buf, True):
+            return buf
+
+    def _stream_is_misconfigured(stream):
+        """A stream is misconfigured if its encoding is ASCII."""
+        # If the stream does not have an encoding set, we assume it's set
+        # to ASCII.  This appears to happen in certain unittest
+        # environments.  It's not quite clear what the correct behavior is
+        # but this at least will force Click to recover somehow.
+        return is_ascii_encoding(getattr(stream, 'encoding', None) or 'ascii')
+
+    def _is_compatible_text_stream(stream, encoding, errors):
+        stream_encoding = getattr(stream, 'encoding', None)
+        stream_errors = getattr(stream, 'errors', None)
+
+        # Perfect match.
+        if stream_encoding == encoding and stream_errors == errors:
+            return True
+
+        # Otherwise, it's only a compatible stream if we did not ask for
+        # an encoding.
+        if encoding is None:
+            return stream_encoding is not None
+
+        return False
+
+    def _force_correct_text_reader(text_reader, encoding, errors,
+                                   force_readable=False):
+        if _is_binary_reader(text_reader, False):
+            binary_reader = text_reader
+        else:
+            # If there is no target encoding set, we need to verify that the
+            # reader is not actually misconfigured.
+            if encoding is None and not _stream_is_misconfigured(text_reader):
+                return text_reader
+
+            if _is_compatible_text_stream(text_reader, encoding, errors):
+                return text_reader
+
+            # If the reader has no encoding, we try to find the underlying
+            # binary reader for it.  If that fails because the environment is
+            # misconfigured, we silently go with the same reader because this
+            # is too common to happen.  In that case, mojibake is better than
+            # exceptions.
+            binary_reader = _find_binary_reader(text_reader)
+            if binary_reader is None:
+                return text_reader
+
+        # At this point, we default the errors to replace instead of strict
+        # because nobody handles those errors anyways and at this point
+        # we're so fundamentally fucked that nothing can repair it.
+        if errors is None:
+            errors = 'replace'
+        return _make_text_stream(binary_reader, encoding, errors,
+                                 force_readable=force_readable)
+
+    def _force_correct_text_writer(text_writer, encoding, errors,
+                                   force_writable=False):
+        if _is_binary_writer(text_writer, False):
+            binary_writer = text_writer
+        else:
+            # If there is no target encoding set, we need to verify that the
+            # writer is not actually misconfigured.
+            if encoding is None and not _stream_is_misconfigured(text_writer):
+                return text_writer
+
+            if _is_compatible_text_stream(text_writer, encoding, errors):
+                return text_writer
+
+            # If the writer has no encoding, we try to find the underlying
+            # binary writer for it.  If that fails because the environment is
+            # misconfigured, we silently go with the same writer because this
+            # is too common to happen.  In that case, mojibake is better than
+            # exceptions.
+            binary_writer = _find_binary_writer(text_writer)
+            if binary_writer is None:
+                return text_writer
+
+        # At this point, we default the errors to replace instead of strict
+        # because nobody handles those errors anyways and at this point
+        # we're so fundamentally fucked that nothing can repair it.
+        if errors is None:
+            errors = 'replace'
+        return _make_text_stream(binary_writer, encoding, errors,
+                                 force_writable=force_writable)
+
+    def get_binary_stdin():
+        reader = _find_binary_reader(sys.stdin)
+        if reader is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stdin.')
+        return reader
+
+    def get_binary_stdout():
+        writer = _find_binary_writer(sys.stdout)
+        if writer is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stdout.')
+        return writer
+
+    def get_binary_stderr():
+        writer = _find_binary_writer(sys.stderr)
+        if writer is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stderr.')
+        return writer
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_reader(sys.stdin, encoding, errors,
+                                          force_readable=True)
+
+    def get_text_stdout(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(sys.stdout, encoding, errors,
+                                          force_writable=True)
+
+    def get_text_stderr(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(sys.stderr, encoding, errors,
+                                          force_writable=True)
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), 'replace')
+        else:
+            value = value.encode('utf-8', 'surrogateescape') \
+                .decode('utf-8', 'replace')
+        return value
+
+
+def get_streerror(e, default=None):
+    if hasattr(e, 'strerror'):
+        msg = e.strerror
+    else:
+        if default is not None:
+            msg = default
+        else:
+            msg = str(e)
+    if isinstance(msg, bytes):
+        msg = msg.decode('utf-8', 'replace')
+    return msg
+
+
+def open_stream(filename, mode='r', encoding=None, errors='strict',
+                atomic=False):
+    # Standard streams first.  These are simple because they don't need
+    # special handling for the atomic flag.  It's entirely ignored.
+    if filename == '-':
+        if any(m in mode for m in ['w', 'a', 'x']):
+            if 'b' in mode:
+                return get_binary_stdout(), False
+            return get_text_stdout(encoding=encoding, errors=errors), False
+        if 'b' in mode:
+            return get_binary_stdin(), False
+        return get_text_stdin(encoding=encoding, errors=errors), False
+
+    # Non-atomic writes directly go out through the regular open functions.
+    if not atomic:
+        if encoding is None:
+            return open(filename, mode), True
+        return io.open(filename, mode, encoding=encoding, errors=errors), True
+
+    # Some usability stuff for atomic writes
+    if 'a' in mode:
+        raise ValueError(
+            'Appending to an existing file is not supported, because that '
+            'would involve an expensive `copy`-operation to a temporary '
+            'file. Open the file in normal `w`-mode and copy explicitly '
+            'if that\'s what you\'re after.'
+        )
+    if 'x' in mode:
+        raise ValueError('Use the `overwrite`-parameter instead.')
+    if 'w' not in mode:
+        raise ValueError('Atomic writes only make sense with `w`-mode.')
+
+    # Atomic writes are more complicated.  They work by opening a file
+    # as a proxy in the same folder and then using the fdopen
+    # functionality to wrap it in a Python file.  Then we wrap it in an
+    # atomic file that moves the file over on close.
+    import tempfile
+    fd, tmp_filename = tempfile.mkstemp(dir=os.path.dirname(filename),
+                                        prefix='.__atomic-write')
+
+    if encoding is not None:
+        f = io.open(fd, mode, encoding=encoding, errors=errors)
+    else:
+        f = os.fdopen(fd, mode)
+
+    return _AtomicFile(f, tmp_filename, os.path.realpath(filename)), True
+
+
+# Used in a destructor call, needs extra protection from interpreter cleanup.
+if hasattr(os, 'replace'):
+    _replace = os.replace
+    _can_replace = True
+else:
+    _replace = os.rename
+    _can_replace = not WIN
+
+
+class _AtomicFile(object):
+
+    def __init__(self, f, tmp_filename, real_filename):
+        self._f = f
+        self._tmp_filename = tmp_filename
+        self._real_filename = real_filename
+        self.closed = False
+
+    @property
+    def name(self):
+        return self._real_filename
+
+    def close(self, delete=False):
+        if self.closed:
+            return
+        self._f.close()
+        if not _can_replace:
+            try:
+                os.remove(self._real_filename)
+            except OSError:
+                pass
+        _replace(self._tmp_filename, self._real_filename)
+        self.closed = True
+
+    def __getattr__(self, name):
+        return getattr(self._f, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close(delete=exc_type is not None)
+
+    def __repr__(self):
+        return repr(self._f)
+
+
+auto_wrap_for_ansi = None
+colorama = None
+get_winterm_size = None
+
+
+def strip_ansi(value):
+    return _ansi_re.sub('', value)
+
+
+def should_strip_ansi(stream=None, color=None):
+    if color is None:
+        if stream is None:
+            stream = sys.stdin
+        return not isatty(stream)
+    return not color
+
+
+# If we're on Windows, we provide transparent integration through
+# colorama.  This will make ANSI colors through the echo function
+# work automatically.
+if WIN:
+    # Windows has a smaller terminal
+    DEFAULT_COLUMNS = 79
+
+    from ._winconsole import _get_windows_console_stream, _wrap_std_stream
+
+    def _get_argv_encoding():
+        import locale
+        return locale.getpreferredencoding()
+
+    if PY2:
+        def raw_input(prompt=''):
+            sys.stderr.flush()
+            if prompt:
+                stdout = _default_text_stdout()
+                stdout.write(prompt)
+            stdin = _default_text_stdin()
+            return stdin.readline().rstrip('\r\n')
+
+    try:
+        import colorama
+    except ImportError:
+        pass
+    else:
+        _ansi_stream_wrappers = WeakKeyDictionary()
+
+        def auto_wrap_for_ansi(stream, color=None):
+            """This function wraps a stream so that calls through colorama
+            are issued to the win32 console API to recolor on demand.  It
+            also ensures to reset the colors if a write call is interrupted
+            to not destroy the console afterwards.
+            """
+            try:
+                cached = _ansi_stream_wrappers.get(stream)
+            except Exception:
+                cached = None
+            if cached is not None:
+                return cached
+            strip = should_strip_ansi(stream, color)
+            ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip)
+            rv = ansi_wrapper.stream
+            _write = rv.write
+
+            def _safe_write(s):
+                try:
+                    return _write(s)
+                except:
+                    ansi_wrapper.reset_all()
+                    raise
+
+            rv.write = _safe_write
+            try:
+                _ansi_stream_wrappers[stream] = rv
+            except Exception:
+                pass
+            return rv
+
+        def get_winterm_size():
+            win = colorama.win32.GetConsoleScreenBufferInfo(
+                colorama.win32.STDOUT).srWindow
+            return win.Right - win.Left, win.Bottom - win.Top
+else:
+    def _get_argv_encoding():
+        return getattr(sys.stdin, 'encoding', None) or get_filesystem_encoding()
+
+    _get_windows_console_stream = lambda *x: None
+    _wrap_std_stream = lambda *x: None
+
+
+def term_len(x):
+    return len(strip_ansi(x))
+
+
+def isatty(stream):
+    try:
+        return stream.isatty()
+    except Exception:
+        return False
+
+
+def _make_cached_stream_func(src_func, wrapper_func):
+    cache = WeakKeyDictionary()
+    def func():
+        stream = src_func()
+        try:
+            rv = cache.get(stream)
+        except Exception:
+            rv = None
+        if rv is not None:
+            return rv
+        rv = wrapper_func()
+        try:
+            stream = src_func()  # In case wrapper_func() modified the stream
+            cache[stream] = rv
+        except Exception:
+            pass
+        return rv
+    return func
+
+
+_default_text_stdin = _make_cached_stream_func(
+    lambda: sys.stdin, get_text_stdin)
+_default_text_stdout = _make_cached_stream_func(
+    lambda: sys.stdout, get_text_stdout)
+_default_text_stderr = _make_cached_stream_func(
+    lambda: sys.stderr, get_text_stderr)
+
+
+binary_streams = {
+    'stdin': get_binary_stdin,
+    'stdout': get_binary_stdout,
+    'stderr': get_binary_stderr,
+}
+
+text_streams = {
+    'stdin': get_text_stdin,
+    'stdout': get_text_stdout,
+    'stderr': get_text_stderr,
+}

python/click/_termui_impl.py

@@ -0,0 +1,621 @@
+# -*- coding: utf-8 -*-
+"""
+click._termui_impl
+~~~~~~~~~~~~~~~~~~
+
+This module contains implementations for the termui module. To keep the
+import time of Click down, some infrequently used functionality is
+placed in this module and only imported as needed.
+
+:copyright: © 2014 by the Pallets team.
+:license: BSD, see LICENSE.rst for more details.
+"""
+
+import os
+import sys
+import time
+import math
+import contextlib
+from ._compat import _default_text_stdout, range_type, PY2, isatty, \
+     open_stream, strip_ansi, term_len, get_best_encoding, WIN, int_types, \
+     CYGWIN
+from .utils import echo
+from .exceptions import ClickException
+
+
+if os.name == 'nt':
+    BEFORE_BAR = '\r'
+    AFTER_BAR = '\n'
+else:
+    BEFORE_BAR = '\r\033[?25l'
+    AFTER_BAR = '\033[?25h\n'
+
+
+def _length_hint(obj):
+    """Returns the length hint of an object."""
+    try:
+        return len(obj)
+    except (AttributeError, TypeError):
+        try:
+            get_hint = type(obj).__length_hint__
+        except AttributeError:
+            return None
+        try:
+            hint = get_hint(obj)
+        except TypeError:
+            return None
+        if hint is NotImplemented or \
+           not isinstance(hint, int_types) or \
+           hint < 0:
+            return None
+        return hint
+
+
+class ProgressBar(object):
+
+    def __init__(self, iterable, length=None, fill_char='#', empty_char=' ',
+                 bar_template='%(bar)s', info_sep='  ', show_eta=True,
+                 show_percent=None, show_pos=False, item_show_func=None,
+                 label=None, file=None, color=None, width=30):
+        self.fill_char = fill_char
+        self.empty_char = empty_char
+        self.bar_template = bar_template
+        self.info_sep = info_sep
+        self.show_eta = show_eta
+        self.show_percent = show_percent
+        self.show_pos = show_pos
+        self.item_show_func = item_show_func
+        self.label = label or ''
+        if file is None:
+            file = _default_text_stdout()
+        self.file = file
+        self.color = color
+        self.width = width
+        self.autowidth = width == 0
+
+        if length is None:
+            length = _length_hint(iterable)
+        if iterable is None:
+            if length is None:
+                raise TypeError('iterable or length is required')
+            iterable = range_type(length)
+        self.iter = iter(iterable)
+        self.length = length
+        self.length_known = length is not None
+        self.pos = 0
+        self.avg = []
+        self.start = self.last_eta = time.time()
+        self.eta_known = False
+        self.finished = False
+        self.max_width = None
+        self.entered = False
+        self.current_item = None
+        self.is_hidden = not isatty(self.file)
+        self._last_line = None
+        self.short_limit = 0.5
+
+    def __enter__(self):
+        self.entered = True
+        self.render_progress()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.render_finish()
+
+    def __iter__(self):
+        if not self.entered:
+            raise RuntimeError('You need to use progress bars in a with block.')
+        self.render_progress()
+        return self.generator()
+
+    def is_fast(self):
+        return time.time() - self.start <= self.short_limit
+
+    def render_finish(self):
+        if self.is_hidden or self.is_fast():
+            return
+        self.file.write(AFTER_BAR)
+        self.file.flush()
+
+    @property
+    def pct(self):
+        if self.finished:
+            return 1.0
+        return min(self.pos / (float(self.length) or 1), 1.0)
+
+    @property
+    def time_per_iteration(self):
+        if not self.avg:
+            return 0.0
+        return sum(self.avg) / float(len(self.avg))
+
+    @property
+    def eta(self):
+        if self.length_known and not self.finished:
+            return self.time_per_iteration * (self.length - self.pos)
+        return 0.0
+
+    def format_eta(self):
+        if self.eta_known:
+            t = int(self.eta)
+            seconds = t % 60
+            t //= 60
+            minutes = t % 60
+            t //= 60
+            hours = t % 24
+            t //= 24
+            if t > 0:
+                days = t
+                return '%dd %02d:%02d:%02d' % (days, hours, minutes, seconds)
+            else:
+                return '%02d:%02d:%02d' % (hours, minutes, seconds)
+        return ''
+
+    def format_pos(self):
+        pos = str(self.pos)
+        if self.length_known:
+            pos += '/%s' % self.length
+        return pos
+
+    def format_pct(self):
+        return ('% 4d%%' % int(self.pct * 100))[1:]
+
+    def format_bar(self):
+        if self.length_known:
+            bar_length = int(self.pct * self.width)
+            bar = self.fill_char * bar_length
+            bar += self.empty_char * (self.width - bar_length)
+        elif self.finished:
+            bar = self.fill_char * self.width
+        else:
+            bar = list(self.empty_char * (self.width or 1))
+            if self.time_per_iteration != 0:
+                bar[int((math.cos(self.pos * self.time_per_iteration)
+                    / 2.0 + 0.5) * self.width)] = self.fill_char
+            bar = ''.join(bar)
+        return bar
+
+    def format_progress_line(self):
+        show_percent = self.show_percent
+
+        info_bits = []
+        if self.length_known and show_percent is None:
+            show_percent = not self.show_pos
+
+        if self.show_pos:
+            info_bits.append(self.format_pos())
+        if show_percent:
+            info_bits.append(self.format_pct())
+        if self.show_eta and self.eta_known and not self.finished:
+            info_bits.append(self.format_eta())
+        if self.item_show_func is not None:
+            item_info = self.item_show_func(self.current_item)
+            if item_info is not None:
+                info_bits.append(item_info)
+
+        return (self.bar_template % {
+            'label': self.label,
+            'bar': self.format_bar(),
+            'info': self.info_sep.join(info_bits)
+        }).rstrip()
+
+    def render_progress(self):
+        from .termui import get_terminal_size
+
+        if self.is_hidden:
+            return
+
+        buf = []
+        # Update width in case the terminal has been resized
+        if self.autowidth:
+            old_width = self.width
+            self.width = 0
+            clutter_length = term_len(self.format_progress_line())
+            new_width = max(0, get_terminal_size()[0] - clutter_length)
+            if new_width < old_width:
+                buf.append(BEFORE_BAR)
+                buf.append(' ' * self.max_width)
+                self.max_width = new_width
+            self.width = new_width
+
+        clear_width = self.width
+        if self.max_width is not None:
+            clear_width = self.max_width
+
+        buf.append(BEFORE_BAR)
+        line = self.format_progress_line()
+        line_len = term_len(line)
+        if self.max_width is None or self.max_width < line_len:
+            self.max_width = line_len
+
+        buf.append(line)
+        buf.append(' ' * (clear_width - line_len))
+        line = ''.join(buf)
+        # Render the line only if it changed.
+
+        if line != self._last_line and not self.is_fast():
+            self._last_line = line
+            echo(line, file=self.file, color=self.color, nl=False)
+            self.file.flush()
+
+    def make_step(self, n_steps):
+        self.pos += n_steps
+        if self.length_known and self.pos >= self.length:
+            self.finished = True
+
+        if (time.time() - self.last_eta) < 1.0:
+            return
+
+        self.last_eta = time.time()
+
+        # self.avg is a rolling list of length <= 7 of steps where steps are
+        # defined as time elapsed divided by the total progress through
+        # self.length.
+        if self.pos:
+            step = (time.time() - self.start) / self.pos
+        else:
+            step = time.time() - self.start
+
+        self.avg = self.avg[-6:] + [step]
+
+        self.eta_known = self.length_known
+
+    def update(self, n_steps):
+        self.make_step(n_steps)
+        self.render_progress()
+
+    def finish(self):
+        self.eta_known = 0
+        self.current_item = None
+        self.finished = True
+
+    def generator(self):
+        """
+        Returns a generator which yields the items added to the bar during
+        construction, and updates the progress bar *after* the yielded block
+        returns.
+        """
+        if not self.entered:
+            raise RuntimeError('You need to use progress bars in a with block.')
+
+        if self.is_hidden:
+            for rv in self.iter:
+                yield rv
+        else:
+            for rv in self.iter:
+                self.current_item = rv
+                yield rv
+                self.update(1)
+            self.finish()
+            self.render_progress()
+
+
+def pager(generator, color=None):
+    """Decide what method to use for paging through text."""
+    stdout = _default_text_stdout()
+    if not isatty(sys.stdin) or not isatty(stdout):
+        return _nullpager(stdout, generator, color)
+    pager_cmd = (os.environ.get('PAGER', None) or '').strip()
+    if pager_cmd:
+        if WIN:
+            return _tempfilepager(generator, pager_cmd, color)
+        return _pipepager(generator, pager_cmd, color)
+    if os.environ.get('TERM') in ('dumb', 'emacs'):
+        return _nullpager(stdout, generator, color)
+    if WIN or sys.platform.startswith('os2'):
+        return _tempfilepager(generator, 'more <', color)
+    if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
+        return _pipepager(generator, 'less', color)
+
+    import tempfile
+    fd, filename = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
+            return _pipepager(generator, 'more', color)
+        return _nullpager(stdout, generator, color)
+    finally:
+        os.unlink(filename)
+
+
+def _pipepager(generator, cmd, color):
+    """Page through text by feeding it to another program.  Invoking a
+    pager through this might support colors.
+    """
+    import subprocess
+    env = dict(os.environ)
+
+    # If we're piping to less we might support colors under the
+    # condition that
+    cmd_detail = cmd.rsplit('/', 1)[-1].split()
+    if color is None and cmd_detail[0] == 'less':
+        less_flags = os.environ.get('LESS', '') + ' '.join(cmd_detail[1:])
+        if not less_flags:
+            env['LESS'] = '-R'
+            color = True
+        elif 'r' in less_flags or 'R' in less_flags:
+            color = True
+
+    c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+                         env=env)
+    encoding = get_best_encoding(c.stdin)
+    try:
+        for text in generator:
+            if not color:
+                text = strip_ansi(text)
+
+            c.stdin.write(text.encode(encoding, 'replace'))
+    except (IOError, KeyboardInterrupt):
+        pass
+    else:
+        c.stdin.close()
+
+    # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
+    # search or other commands inside less).
+    #
+    # That means when the user hits ^C, the parent process (click) terminates,
+    # but less is still alive, paging the output and messing up the terminal.
+    #
+    # If the user wants to make the pager exit on ^C, they should set
+    # `LESS='-K'`. It's not our decision to make.
+    while True:
+        try:
+            c.wait()
+        except KeyboardInterrupt:
+            pass
+        else:
+            break
+
+
+def _tempfilepager(generator, cmd, color):
+    """Page through text by invoking a program on a temporary file."""
+    import tempfile
+    filename = tempfile.mktemp()
+    # TODO: This never terminates if the passed generator never terminates.
+    text = "".join(generator)
+    if not color:
+        text = strip_ansi(text)
+    encoding = get_best_encoding(sys.stdout)
+    with open_stream(filename, 'wb')[0] as f:
+        f.write(text.encode(encoding))
+    try:
+        os.system(cmd + ' "' + filename + '"')
+    finally:
+        os.unlink(filename)
+
+
+def _nullpager(stream, generator, color):
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    for text in generator:
+        if not color:
+            text = strip_ansi(text)
+        stream.write(text)
+
+
+class Editor(object):
+
+    def __init__(self, editor=None, env=None, require_save=True,
+                 extension='.txt'):
+        self.editor = editor
+        self.env = env
+        self.require_save = require_save
+        self.extension = extension
+
+    def get_editor(self):
+        if self.editor is not None:
+            return self.editor
+        for key in 'VISUAL', 'EDITOR':
+            rv = os.environ.get(key)
+            if rv:
+                return rv
+        if WIN:
+            return 'notepad'
+        for editor in 'vim', 'nano':
+            if os.system('which %s >/dev/null 2>&1' % editor) == 0:
+                return editor
+        return 'vi'
+
+    def edit_file(self, filename):
+        import subprocess
+        editor = self.get_editor()
+        if self.env:
+            environ = os.environ.copy()
+            environ.update(self.env)
+        else:
+            environ = None
+        try:
+            c = subprocess.Popen('%s "%s"' % (editor, filename),
+                                 env=environ, shell=True)
+            exit_code = c.wait()
+            if exit_code != 0:
+                raise ClickException('%s: Editing failed!' % editor)
+        except OSError as e:
+            raise ClickException('%s: Editing failed: %s' % (editor, e))
+
+    def edit(self, text):
+        import tempfile
+
+        text = text or ''
+        if text and not text.endswith('\n'):
+            text += '\n'
+
+        fd, name = tempfile.mkstemp(prefix='editor-', suffix=self.extension)
+        try:
+            if WIN:
+                encoding = 'utf-8-sig'
+                text = text.replace('\n', '\r\n')
+            else:
+                encoding = 'utf-8'
+            text = text.encode(encoding)
+
+            f = os.fdopen(fd, 'wb')
+            f.write(text)
+            f.close()
+            timestamp = os.path.getmtime(name)
+
+            self.edit_file(name)
+
+            if self.require_save \
+               and os.path.getmtime(name) == timestamp:
+                return None
+
+            f = open(name, 'rb')
+            try:
+                rv = f.read()
+            finally:
+                f.close()
+            return rv.decode('utf-8-sig').replace('\r\n', '\n')
+        finally:
+            os.unlink(name)
+
+
+def open_url(url, wait=False, locate=False):
+    import subprocess
+
+    def _unquote_file(url):
+        try:
+            import urllib
+        except ImportError:
+            import urllib
+        if url.startswith('file://'):
+            url = urllib.unquote(url[7:])
+        return url
+
+    if sys.platform == 'darwin':
+        args = ['open']
+        if wait:
+            args.append('-W')
+        if locate:
+            args.append('-R')
+        args.append(_unquote_file(url))
+        null = open('/dev/null', 'w')
+        try:
+            return subprocess.Popen(args, stderr=null).wait()
+        finally:
+            null.close()
+    elif WIN:
+        if locate:
+            url = _unquote_file(url)
+            args = 'explorer /select,"%s"' % _unquote_file(
+                url.replace('"', ''))
+        else:
+            args = 'start %s "" "%s"' % (
+                wait and '/WAIT' or '', url.replace('"', ''))
+        return os.system(args)
+    elif CYGWIN:
+        if locate:
+            url = _unquote_file(url)
+            args = 'cygstart "%s"' % (os.path.dirname(url).replace('"', ''))
+        else:
+            args = 'cygstart %s "%s"' % (
+                wait and '-w' or '', url.replace('"', ''))
+        return os.system(args)
+
+    try:
+        if locate:
+            url = os.path.dirname(_unquote_file(url)) or '.'
+        else:
+            url = _unquote_file(url)
+        c = subprocess.Popen(['xdg-open', url])
+        if wait:
+            return c.wait()
+        return 0
+    except OSError:
+        if url.startswith(('http://', 'https://')) and not locate and not wait:
+            import webbrowser
+            webbrowser.open(url)
+            return 0
+        return 1
+
+
+def _translate_ch_to_exc(ch):
+    if ch == u'\x03':
+        raise KeyboardInterrupt()
+    if ch == u'\x04' and not WIN:  # Unix-like, Ctrl+D
+        raise EOFError()
+    if ch == u'\x1a' and WIN:      # Windows, Ctrl+Z
+        raise EOFError()
+
+
+if WIN:
+    import msvcrt
+
+    @contextlib.contextmanager
+    def raw_terminal():
+        yield
+
+    def getchar(echo):
+        # The function `getch` will return a bytes object corresponding to
+        # the pressed character. Since Windows 10 build 1803, it will also
+        # return \x00 when called a second time after pressing a regular key.
+        #
+        # `getwch` does not share this probably-bugged behavior. Moreover, it
+        # returns a Unicode object by default, which is what we want.
+        #
+        # Either of these functions will return \x00 or \xe0 to indicate
+        # a special key, and you need to call the same function again to get
+        # the "rest" of the code. The fun part is that \u00e0 is
+        # "latin small letter a with grave", so if you type that on a French
+        # keyboard, you _also_ get a \xe0.
+        # E.g., consider the Up arrow. This returns \xe0 and then \x48. The
+        # resulting Unicode string reads as "a with grave" + "capital H".
+        # This is indistinguishable from when the user actually types
+        # "a with grave" and then "capital H".
+        #
+        # When \xe0 is returned, we assume it's part of a special-key sequence
+        # and call `getwch` again, but that means that when the user types
+        # the \u00e0 character, `getchar` doesn't return until a second
+        # character is typed.
+        # The alternative is returning immediately, but that would mess up
+        # cross-platform handling of arrow keys and others that start with
+        # \xe0. Another option is using `getch`, but then we can't reliably
+        # read non-ASCII characters, because return values of `getch` are
+        # limited to the current 8-bit codepage.
+        #
+        # Anyway, Click doesn't claim to do this Right(tm), and using `getwch`
+        # is doing the right thing in more situations than with `getch`.
+        if echo:
+            func = msvcrt.getwche
+        else:
+            func = msvcrt.getwch
+
+        rv = func()
+        if rv in (u'\x00', u'\xe0'):
+            # \x00 and \xe0 are control characters that indicate special key,
+            # see above.
+            rv += func()
+        _translate_ch_to_exc(rv)
+        return rv
+else:
+    import tty
+    import termios
+
+    @contextlib.contextmanager
+    def raw_terminal():
+        if not isatty(sys.stdin):
+            f = open('/dev/tty')
+            fd = f.fileno()
+        else:
+            fd = sys.stdin.fileno()
+            f = None
+        try:
+            old_settings = termios.tcgetattr(fd)
+            try:
+                tty.setraw(fd)
+                yield fd
+            finally:
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                sys.stdout.flush()
+                if f is not None:
+                    f.close()
+        except termios.error:
+            pass
+
+    def getchar(echo):
+        with raw_terminal() as fd:
+            ch = os.read(fd, 32)
+            ch = ch.decode(get_best_encoding(sys.stdin), 'replace')
+            if echo and isatty(sys.stdout):
+                sys.stdout.write(ch)
+            _translate_ch_to_exc(ch)
+            return ch

python/click/_textwrap.py

@@ -0,0 +1,38 @@
+import textwrap
+from contextlib import contextmanager
+
+
+class TextWrapper(textwrap.TextWrapper):
+
+    def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
+        space_left = max(width - cur_len, 1)
+
+        if self.break_long_words:
+            last = reversed_chunks[-1]
+            cut = last[:space_left]
+            res = last[space_left:]
+            cur_line.append(cut)
+            reversed_chunks[-1] = res
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+    @contextmanager
+    def extra_indent(self, indent):
+        old_initial_indent = self.initial_indent
+        old_subsequent_indent = self.subsequent_indent
+        self.initial_indent += indent
+        self.subsequent_indent += indent
+        try:
+            yield
+        finally:
+            self.initial_indent = old_initial_indent
+            self.subsequent_indent = old_subsequent_indent
+
+    def indent_only(self, text):
+        rv = []
+        for idx, line in enumerate(text.splitlines()):
+            indent = self.initial_indent
+            if idx > 0:
+                indent = self.subsequent_indent
+            rv.append(indent + line)
+        return '\n'.join(rv)

python/click/_unicodefun.py

@@ -0,0 +1,125 @@
+import os
+import sys
+import codecs
+
+from ._compat import PY2
+
+
+# If someone wants to vendor click, we want to ensure the
+# correct package is discovered.  Ideally we could use a
+# relative import here but unfortunately Python does not
+# support that.
+click = sys.modules[__name__.rsplit('.', 1)[0]]
+
+
+def _find_unicode_literals_frame():
+    import __future__
+    if not hasattr(sys, '_getframe'):  # not all Python implementations have it
+        return 0
+    frm = sys._getframe(1)
+    idx = 1
+    while frm is not None:
+        if frm.f_globals.get('__name__', '').startswith('click.'):
+            frm = frm.f_back
+            idx += 1
+        elif frm.f_code.co_flags & __future__.unicode_literals.compiler_flag:
+            return idx
+        else:
+            break
+    return 0
+
+
+def _check_for_unicode_literals():
+    if not __debug__:
+        return
+    if not PY2 or click.disable_unicode_literals_warning:
+        return
+    bad_frame = _find_unicode_literals_frame()
+    if bad_frame <= 0:
+        return
+    from warnings import warn
+    warn(Warning('Click detected the use of the unicode_literals '
+                 '__future__ import.  This is heavily discouraged '
+                 'because it can introduce subtle bugs in your '
+                 'code.  You should instead use explicit u"" literals '
+                 'for your unicode strings.  For more information see '
+                 'https://click.palletsprojects.com/python3/'),
+         stacklevel=bad_frame)
+
+
+def _verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    if PY2:
+        return
+    try:
+        import locale
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = 'ascii'
+    if fs_enc != 'ascii':
+        return
+
+    extra = ''
+    if os.name == 'posix':
+        import subprocess
+        try:
+            rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE).communicate()[0]
+        except OSError:
+            rv = b''
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode('ascii', 'replace')
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith(('.utf-8', '.utf8')):
+                good_locales.add(locale)
+                if locale.lower() in ('c.utf8', 'c.utf-8'):
+                    has_c_utf8 = True
+
+        extra += '\n\n'
+        if not good_locales:
+            extra += (
+                'Additional information: on this system no suitable UTF-8\n'
+                'locales were discovered.  This most likely requires resolving\n'
+                'by reconfiguring the locale system.'
+            )
+        elif has_c_utf8:
+            extra += (
+                'This system supports the C.UTF-8 locale which is recommended.\n'
+                'You might be able to resolve your issue by exporting the\n'
+                'following environment variables:\n\n'
+                '    export LC_ALL=C.UTF-8\n'
+                '    export LANG=C.UTF-8'
+            )
+        else:
+            extra += (
+                'This system lists a couple of UTF-8 supporting locales that\n'
+                'you can pick from.  The following suitable locales were\n'
+                'discovered: %s'
+            ) % ', '.join(sorted(good_locales))
+
+        bad_locale = None
+        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
+            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                '\n\nClick discovered that you exported a UTF-8 locale\n'
+                'but the locale system could not pick up from it because\n'
+                'it does not exist.  The exported locale is "%s" but it\n'
+                'is not supported'
+            ) % bad_locale
+
+    raise RuntimeError(
+        'Click will abort further execution because Python 3 was'
+        ' configured to use ASCII as encoding for the environment.'
+        ' Consult https://click.palletsprojects.com/en/7.x/python3/ for'
+        ' mitigation steps.' + extra
+    )

python/click/_winconsole.py

@@ -0,0 +1,307 @@
+# -*- coding: utf-8 -*-
+# This module is based on the excellent work by Adam Bartoš who
+# provided a lot of what went into the implementation here in
+# the discussion to issue1602 in the Python bug tracker.
+#
+# There are some general differences in regards to how this works
+# compared to the original patches as we do not need to patch
+# the entire interpreter but just work in our little world of
+# echo and prmopt.
+
+import io
+import os
+import sys
+import zlib
+import time
+import ctypes
+import msvcrt
+from ._compat import _NonClosingTextIOWrapper, text_type, PY2
+from ctypes import byref, POINTER, c_int, c_char, c_char_p, \
+     c_void_p, py_object, c_ssize_t, c_ulong, windll, WINFUNCTYPE
+try:
+    from ctypes import pythonapi
+    PyObject_GetBuffer = pythonapi.PyObject_GetBuffer
+    PyBuffer_Release = pythonapi.PyBuffer_Release
+except ImportError:
+    pythonapi = None
+from ctypes.wintypes import LPWSTR, LPCWSTR
+
+
+c_ssize_p = POINTER(c_ssize_t)
+
+kernel32 = windll.kernel32
+GetStdHandle = kernel32.GetStdHandle
+ReadConsoleW = kernel32.ReadConsoleW
+WriteConsoleW = kernel32.WriteConsoleW
+GetLastError = kernel32.GetLastError
+GetCommandLineW = WINFUNCTYPE(LPWSTR)(
+    ('GetCommandLineW', windll.kernel32))
+CommandLineToArgvW = WINFUNCTYPE(
+    POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
+        ('CommandLineToArgvW', windll.shell32))
+
+
+STDIN_HANDLE = GetStdHandle(-10)
+STDOUT_HANDLE = GetStdHandle(-11)
+STDERR_HANDLE = GetStdHandle(-12)
+
+
+PyBUF_SIMPLE = 0
+PyBUF_WRITABLE = 1
+
+ERROR_SUCCESS = 0
+ERROR_NOT_ENOUGH_MEMORY = 8
+ERROR_OPERATION_ABORTED = 995
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+EOF = b'\x1a'
+MAX_BYTES_WRITTEN = 32767
+
+
+class Py_buffer(ctypes.Structure):
+    _fields_ = [
+        ('buf', c_void_p),
+        ('obj', py_object),
+        ('len', c_ssize_t),
+        ('itemsize', c_ssize_t),
+        ('readonly', c_int),
+        ('ndim', c_int),
+        ('format', c_char_p),
+        ('shape', c_ssize_p),
+        ('strides', c_ssize_p),
+        ('suboffsets', c_ssize_p),
+        ('internal', c_void_p)
+    ]
+
+    if PY2:
+        _fields_.insert(-1, ('smalltable', c_ssize_t * 2))
+
+
+# On PyPy we cannot get buffers so our ability to operate here is
+# serverly limited.
+if pythonapi is None:
+    get_buffer = None
+else:
+    def get_buffer(obj, writable=False):
+        buf = Py_buffer()
+        flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE
+        PyObject_GetBuffer(py_object(obj), byref(buf), flags)
+        try:
+            buffer_type = c_char * buf.len
+            return buffer_type.from_address(buf.buf)
+        finally:
+            PyBuffer_Release(byref(buf))
+
+
+class _WindowsConsoleRawIOBase(io.RawIOBase):
+
+    def __init__(self, handle):
+        self.handle = handle
+
+    def isatty(self):
+        io.RawIOBase.isatty(self)
+        return True
+
+
+class _WindowsConsoleReader(_WindowsConsoleRawIOBase):
+
+    def readable(self):
+        return True
+
+    def readinto(self, b):
+        bytes_to_be_read = len(b)
+        if not bytes_to_be_read:
+            return 0
+        elif bytes_to_be_read % 2:
+            raise ValueError('cannot read odd number of bytes from '
+                             'UTF-16-LE encoded console')
+
+        buffer = get_buffer(b, writable=True)
+        code_units_to_be_read = bytes_to_be_read // 2
+        code_units_read = c_ulong()
+
+        rv = ReadConsoleW(self.handle, buffer, code_units_to_be_read,
+                          byref(code_units_read), None)
+        if GetLastError() == ERROR_OPERATION_ABORTED:
+            # wait for KeyboardInterrupt
+            time.sleep(0.1)
+        if not rv:
+            raise OSError('Windows error: %s' % GetLastError())
+
+        if buffer[0] == EOF:
+            return 0
+        return 2 * code_units_read.value
+
+
+class _WindowsConsoleWriter(_WindowsConsoleRawIOBase):
+
+    def writable(self):
+        return True
+
+    @staticmethod
+    def _get_error_message(errno):
+        if errno == ERROR_SUCCESS:
+            return 'ERROR_SUCCESS'
+        elif errno == ERROR_NOT_ENOUGH_MEMORY:
+            return 'ERROR_NOT_ENOUGH_MEMORY'
+        return 'Windows error %s' % errno
+
+    def write(self, b):
+        bytes_to_be_written = len(b)
+        buf = get_buffer(b)
+        code_units_to_be_written = min(bytes_to_be_written,
+                                       MAX_BYTES_WRITTEN) // 2
+        code_units_written = c_ulong()
+
+        WriteConsoleW(self.handle, buf, code_units_to_be_written,
+                      byref(code_units_written), None)
+        bytes_written = 2 * code_units_written.value
+
+        if bytes_written == 0 and bytes_to_be_written > 0:
+            raise OSError(self._get_error_message(GetLastError()))
+        return bytes_written
+
+
+class ConsoleStream(object):
+
+    def __init__(self, text_stream, byte_stream):
+        self._text_stream = text_stream
+        self.buffer = byte_stream
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def write(self, x):
+        if isinstance(x, text_type):
+            return self._text_stream.write(x)
+        try:
+            self.flush()
+        except Exception:
+            pass
+        return self.buffer.write(x)
+
+    def writelines(self, lines):
+        for line in lines:
+            self.write(line)
+
+    def __getattr__(self, name):
+        return getattr(self._text_stream, name)
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def __repr__(self):
+        return '<ConsoleStream name=%r encoding=%r>' % (
+            self.name,
+            self.encoding,
+        )
+
+
+class WindowsChunkedWriter(object):
+    """
+    Wraps a stream (such as stdout), acting as a transparent proxy for all
+    attribute access apart from method 'write()' which we wrap to write in
+    limited chunks due to a Windows limitation on binary console streams.
+    """
+    def __init__(self, wrapped):
+        # double-underscore everything to prevent clashes with names of
+        # attributes on the wrapped stream object.
+        self.__wrapped = wrapped
+
+    def __getattr__(self, name):
+        return getattr(self.__wrapped, name)
+
+    def write(self, text):
+        total_to_write = len(text)
+        written = 0
+
+        while written < total_to_write:
+            to_write = min(total_to_write - written, MAX_BYTES_WRITTEN)
+            self.__wrapped.write(text[written:written+to_write])
+            written += to_write
+
+
+_wrapped_std_streams = set()
+
+
+def _wrap_std_stream(name):
+    # Python 2 & Windows 7 and below
+    if PY2 and sys.getwindowsversion()[:2] <= (6, 1) and name not in _wrapped_std_streams:
+        setattr(sys, name, WindowsChunkedWriter(getattr(sys, name)))
+        _wrapped_std_streams.add(name)
+
+
+def _get_text_stdin(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stdout(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stderr(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+if PY2:
+    def _hash_py_argv():
+        return zlib.crc32('\x00'.join(sys.argv[1:]))
+
+    _initial_argv_hash = _hash_py_argv()
+
+    def _get_windows_argv():
+        argc = c_int(0)
+        argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+        argv = [argv_unicode[i] for i in range(0, argc.value)]
+
+        if not hasattr(sys, 'frozen'):
+            argv = argv[1:]
+            while len(argv) > 0:
+                arg = argv[0]
+                if not arg.startswith('-') or arg == '-':
+                    break
+                argv = argv[1:]
+                if arg.startswith(('-c', '-m')):
+                    break
+
+        return argv[1:]
+
+
+_stream_factories = {
+    0: _get_text_stdin,
+    1: _get_text_stdout,
+    2: _get_text_stderr,
+}
+
+
+def _get_windows_console_stream(f, encoding, errors):
+    if get_buffer is not None and \
+       encoding in ('utf-16-le', None) \
+       and errors in ('strict', None) and \
+       hasattr(f, 'isatty') and f.isatty():
+        func = _stream_factories.get(f.fileno())
+        if func is not None:
+            if not PY2:
+                f = getattr(f, 'buffer', None)
+                if f is None:
+                    return None
+            else:
+                # If we are on Python 2 we need to set the stream that we
+                # deal with to binary mode as otherwise the exercise if a
+                # bit moot.  The same problems apply as for
+                # get_binary_stdin and friends from _compat.
+                msvcrt.setmode(f.fileno(), os.O_BINARY)
+            return func(f)

python/click/decorators.py

@@ -0,0 +1,311 @@
+import sys
+import inspect
+
+from functools import update_wrapper
+
+from ._compat import iteritems
+from ._unicodefun import _check_for_unicode_literals
+from .utils import echo
+from .globals import get_current_context
+
+
+def pass_context(f):
+    """Marks a callback as wanting to receive the current context
+    object as first argument.
+    """
+    def new_func(*args, **kwargs):
+        return f(get_current_context(), *args, **kwargs)
+    return update_wrapper(new_func, f)
+
+
+def pass_obj(f):
+    """Similar to :func:`pass_context`, but only pass the object on the
+    context onwards (:attr:`Context.obj`).  This is useful if that object
+    represents the state of a nested system.
+    """
+    def new_func(*args, **kwargs):
+        return f(get_current_context().obj, *args, **kwargs)
+    return update_wrapper(new_func, f)
+
+
+def make_pass_decorator(object_type, ensure=False):
+    """Given an object type this creates a decorator that will work
+    similar to :func:`pass_obj` but instead of passing the object of the
+    current context, it will find the innermost context of type
+    :func:`object_type`.
+
+    This generates a decorator that works roughly like this::
+
+        from functools import update_wrapper
+
+        def decorator(f):
+            @pass_context
+            def new_func(ctx, *args, **kwargs):
+                obj = ctx.find_object(object_type)
+                return ctx.invoke(f, obj, *args, **kwargs)
+            return update_wrapper(new_func, f)
+        return decorator
+
+    :param object_type: the type of the object to pass.
+    :param ensure: if set to `True`, a new object will be created and
+                   remembered on the context if it's not there yet.
+    """
+    def decorator(f):
+        def new_func(*args, **kwargs):
+            ctx = get_current_context()
+            if ensure:
+                obj = ctx.ensure_object(object_type)
+            else:
+                obj = ctx.find_object(object_type)
+            if obj is None:
+                raise RuntimeError('Managed to invoke callback without a '
+                                   'context object of type %r existing'
+                                   % object_type.__name__)
+            return ctx.invoke(f, obj, *args, **kwargs)
+        return update_wrapper(new_func, f)
+    return decorator
+
+
+def _make_command(f, name, attrs, cls):
+    if isinstance(f, Command):
+        raise TypeError('Attempted to convert a callback into a '
+                        'command twice.')
+    try:
+        params = f.__click_params__
+        params.reverse()
+        del f.__click_params__
+    except AttributeError:
+        params = []
+    help = attrs.get('help')
+    if help is None:
+        help = inspect.getdoc(f)
+        if isinstance(help, bytes):
+            help = help.decode('utf-8')
+    else:
+        help = inspect.cleandoc(help)
+    attrs['help'] = help
+    _check_for_unicode_literals()
+    return cls(name=name or f.__name__.lower().replace('_', '-'),
+               callback=f, params=params, **attrs)
+
+
+def command(name=None, cls=None, **attrs):
+    r"""Creates a new :class:`Command` and uses the decorated function as
+    callback.  This will also automatically attach all decorated
+    :func:`option`\s and :func:`argument`\s as parameters to the command.
+
+    The name of the command defaults to the name of the function.  If you
+    want to change that, you can pass the intended name as the first
+    argument.
+
+    All keyword arguments are forwarded to the underlying command class.
+
+    Once decorated the function turns into a :class:`Command` instance
+    that can be invoked as a command line utility or be attached to a
+    command :class:`Group`.
+
+    :param name: the name of the command.  This defaults to the function
+                 name with underscores replaced by dashes.
+    :param cls: the command class to instantiate.  This defaults to
+                :class:`Command`.
+    """
+    if cls is None:
+        cls = Command
+    def decorator(f):
+        cmd = _make_command(f, name, attrs, cls)
+        cmd.__doc__ = f.__doc__
+        return cmd
+    return decorator
+
+
+def group(name=None, **attrs):
+    """Creates a new :class:`Group` with a function as callback.  This
+    works otherwise the same as :func:`command` just that the `cls`
+    parameter is set to :class:`Group`.
+    """
+    attrs.setdefault('cls', Group)
+    return command(name, **attrs)
+
+
+def _param_memo(f, param):
+    if isinstance(f, Command):
+        f.params.append(param)
+    else:
+        if not hasattr(f, '__click_params__'):
+            f.__click_params__ = []
+        f.__click_params__.append(param)
+
+
+def argument(*param_decls, **attrs):
+    """Attaches an argument to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Argument`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Argument` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the argument class to instantiate.  This defaults to
+                :class:`Argument`.
+    """
+    def decorator(f):
+        ArgumentClass = attrs.pop('cls', Argument)
+        _param_memo(f, ArgumentClass(param_decls, **attrs))
+        return f
+    return decorator
+
+
+def option(*param_decls, **attrs):
+    """Attaches an option to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Option`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Option` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the option class to instantiate.  This defaults to
+                :class:`Option`.
+    """
+    def decorator(f):
+        # Issue 926, copy attrs, so pre-defined options can re-use the same cls=
+        option_attrs = attrs.copy()
+
+        if 'help' in option_attrs:
+            option_attrs['help'] = inspect.cleandoc(option_attrs['help'])
+        OptionClass = option_attrs.pop('cls', Option)
+        _param_memo(f, OptionClass(param_decls, **option_attrs))
+        return f
+    return decorator
+
+
+def confirmation_option(*param_decls, **attrs):
+    """Shortcut for confirmation prompts that can be ignored by passing
+    ``--yes`` as parameter.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+
+        @click.command()
+        @click.option('--yes', is_flag=True, callback=callback,
+                      expose_value=False, prompt='Do you want to continue?')
+        def dropdb():
+            pass
+    """
+    def decorator(f):
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('callback', callback)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('prompt', 'Do you want to continue?')
+        attrs.setdefault('help', 'Confirm the action without prompting.')
+        return option(*(param_decls or ('--yes',)), **attrs)(f)
+    return decorator
+
+
+def password_option(*param_decls, **attrs):
+    """Shortcut for password prompts.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        @click.command()
+        @click.option('--password', prompt=True, confirmation_prompt=True,
+                      hide_input=True)
+        def changeadmin(password):
+            pass
+    """
+    def decorator(f):
+        attrs.setdefault('prompt', True)
+        attrs.setdefault('confirmation_prompt', True)
+        attrs.setdefault('hide_input', True)
+        return option(*(param_decls or ('--password',)), **attrs)(f)
+    return decorator
+
+
+def version_option(version=None, *param_decls, **attrs):
+    """Adds a ``--version`` option which immediately ends the program
+    printing out the version number.  This is implemented as an eager
+    option that prints the version and exits the program in the callback.
+
+    :param version: the version number to show.  If not provided Click
+                    attempts an auto discovery via setuptools.
+    :param prog_name: the name of the program (defaults to autodetection)
+    :param message: custom message to show instead of the default
+                    (``'%(prog)s, version %(version)s'``)
+    :param others: everything else is forwarded to :func:`option`.
+    """
+    if version is None:
+        if hasattr(sys, '_getframe'):
+            module = sys._getframe(1).f_globals.get('__name__')
+        else:
+            module = ''
+
+    def decorator(f):
+        prog_name = attrs.pop('prog_name', None)
+        message = attrs.pop('message', '%(prog)s, version %(version)s')
+
+        def callback(ctx, param, value):
+            if not value or ctx.resilient_parsing:
+                return
+            prog = prog_name
+            if prog is None:
+                prog = ctx.find_root().info_name
+            ver = version
+            if ver is None:
+                try:
+                    import pkg_resources
+                except ImportError:
+                    pass
+                else:
+                    for dist in pkg_resources.working_set:
+                        scripts = dist.get_entry_map().get('console_scripts') or {}
+                        for script_name, entry_point in iteritems(scripts):
+                            if entry_point.module_name == module:
+                                ver = dist.version
+                                break
+                if ver is None:
+                    raise RuntimeError('Could not determine version')
+            echo(message % {
+                'prog': prog,
+                'version': ver,
+            }, color=ctx.color)
+            ctx.exit()
+
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('is_eager', True)
+        attrs.setdefault('help', 'Show the version and exit.')
+        attrs['callback'] = callback
+        return option(*(param_decls or ('--version',)), **attrs)(f)
+    return decorator
+
+
+def help_option(*param_decls, **attrs):
+    """Adds a ``--help`` option which immediately ends the program
+    printing out the help page.  This is usually unnecessary to add as
+    this is added by default to all commands unless suppressed.
+
+    Like :func:`version_option`, this is implemented as eager option that
+    prints in the callback and exits.
+
+    All arguments are forwarded to :func:`option`.
+    """
+    def decorator(f):
+        def callback(ctx, param, value):
+            if value and not ctx.resilient_parsing:
+                echo(ctx.get_help(), color=ctx.color)
+                ctx.exit()
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('help', 'Show this message and exit.')
+        attrs.setdefault('is_eager', True)
+        attrs['callback'] = callback
+        return option(*(param_decls or ('--help',)), **attrs)(f)
+    return decorator
+
+
+# Circular dependencies between core and decorators
+from .core import Command, Group, Argument, Option

python/click/exceptions.py

@@ -0,0 +1,235 @@
+from ._compat import PY2, filename_to_ui, get_text_stderr
+from .utils import echo
+
+
+def _join_param_hints(param_hint):
+    if isinstance(param_hint, (tuple, list)):
+        return ' / '.join('"%s"' % x for x in param_hint)
+    return param_hint
+
+
+class ClickException(Exception):
+    """An exception that Click can handle and show to the user."""
+
+    #: The exit code for this exception
+    exit_code = 1
+
+    def __init__(self, message):
+        ctor_msg = message
+        if PY2:
+            if ctor_msg is not None:
+                ctor_msg = ctor_msg.encode('utf-8')
+        Exception.__init__(self, ctor_msg)
+        self.message = message
+
+    def format_message(self):
+        return self.message
+
+    def __str__(self):
+        return self.message
+
+    if PY2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.message.encode('utf-8')
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        echo('Error: %s' % self.format_message(), file=file)
+
+
+class UsageError(ClickException):
+    """An internal exception that signals a usage error.  This typically
+    aborts any further handling.
+
+    :param message: the error message to display.
+    :param ctx: optionally the context that caused this error.  Click will
+                fill in the context automatically in some situations.
+    """
+    exit_code = 2
+
+    def __init__(self, message, ctx=None):
+        ClickException.__init__(self, message)
+        self.ctx = ctx
+        self.cmd = self.ctx and self.ctx.command or None
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        color = None
+        hint = ''
+        if (self.cmd is not None and
+                self.cmd.get_help_option(self.ctx) is not None):
+            hint = ('Try "%s %s" for help.\n'
+                    % (self.ctx.command_path, self.ctx.help_option_names[0]))
+        if self.ctx is not None:
+            color = self.ctx.color
+            echo(self.ctx.get_usage() + '\n%s' % hint, file=file, color=color)
+        echo('Error: %s' % self.format_message(), file=file, color=color)
+
+
+class BadParameter(UsageError):
+    """An exception that formats out a standardized error message for a
+    bad parameter.  This is useful when thrown from a callback or type as
+    Click will attach contextual information to it (for instance, which
+    parameter it is).
+
+    .. versionadded:: 2.0
+
+    :param param: the parameter object that caused this error.  This can
+                  be left out, and Click will attach this info itself
+                  if possible.
+    :param param_hint: a string that shows up as parameter name.  This
+                       can be used as alternative to `param` in cases
+                       where custom validation should happen.  If it is
+                       a string it's used as such, if it's a list then
+                       each item is quoted and separated.
+    """
+
+    def __init__(self, message, ctx=None, param=None,
+                 param_hint=None):
+        UsageError.__init__(self, message, ctx)
+        self.param = param
+        self.param_hint = param_hint
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)
+        else:
+            return 'Invalid value: %s' % self.message
+        param_hint = _join_param_hints(param_hint)
+
+        return 'Invalid value for %s: %s' % (param_hint, self.message)
+
+
+class MissingParameter(BadParameter):
+    """Raised if click required an option or argument but it was not
+    provided when invoking the script.
+
+    .. versionadded:: 4.0
+
+    :param param_type: a string that indicates the type of the parameter.
+                       The default is to inherit the parameter type from
+                       the given `param`.  Valid values are ``'parameter'``,
+                       ``'option'`` or ``'argument'``.
+    """
+
+    def __init__(self, message=None, ctx=None, param=None,
+                 param_hint=None, param_type=None):
+        BadParameter.__init__(self, message, ctx, param, param_hint)
+        self.param_type = param_type
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)
+        else:
+            param_hint = None
+        param_hint = _join_param_hints(param_hint)
+
+        param_type = self.param_type
+        if param_type is None and self.param is not None:
+            param_type = self.param.param_type_name
+
+        msg = self.message
+        if self.param is not None:
+            msg_extra = self.param.type.get_missing_message(self.param)
+            if msg_extra:
+                if msg:
+                    msg += '.  ' + msg_extra
+                else:
+                    msg = msg_extra
+
+        return 'Missing %s%s%s%s' % (
+            param_type,
+            param_hint and ' %s' % param_hint or '',
+            msg and '.  ' or '.',
+            msg or '',
+        )
+
+
+class NoSuchOption(UsageError):
+    """Raised if click attempted to handle an option that does not
+    exist.
+
+    .. versionadded:: 4.0
+    """
+
+    def __init__(self, option_name, message=None, possibilities=None,
+                 ctx=None):
+        if message is None:
+            message = 'no such option: %s' % option_name
+        UsageError.__init__(self, message, ctx)
+        self.option_name = option_name
+        self.possibilities = possibilities
+
+    def format_message(self):
+        bits = [self.message]
+        if self.possibilities:
+            if len(self.possibilities) == 1:
+                bits.append('Did you mean %s?' % self.possibilities[0])
+            else:
+                possibilities = sorted(self.possibilities)
+                bits.append('(Possible options: %s)' % ', '.join(possibilities))
+        return '  '.join(bits)
+
+
+class BadOptionUsage(UsageError):
+    """Raised if an option is generally supplied but the use of the option
+    was incorrect.  This is for instance raised if the number of arguments
+    for an option is not correct.
+
+    .. versionadded:: 4.0
+
+    :param option_name: the name of the option being used incorrectly.
+    """
+
+    def __init__(self, option_name, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+        self.option_name = option_name
+
+
+class BadArgumentUsage(UsageError):
+    """Raised if an argument is generally supplied but the use of the argument
+    was incorrect.  This is for instance raised if the number of values
+    for an argument is not correct.
+
+    .. versionadded:: 6.0
+    """
+
+    def __init__(self, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+
+
+class FileError(ClickException):
+    """Raised if a file cannot be opened."""
+
+    def __init__(self, filename, hint=None):
+        ui_filename = filename_to_ui(filename)
+        if hint is None:
+            hint = 'unknown error'
+        ClickException.__init__(self, hint)
+        self.ui_filename = ui_filename
+        self.filename = filename
+
+    def format_message(self):
+        return 'Could not open file %s: %s' % (self.ui_filename, self.message)
+
+
+class Abort(RuntimeError):
+    """An internal signalling exception that signals Click to abort."""
+
+
+class Exit(RuntimeError):
+    """An exception that indicates that the application should exit with some
+    status code.
+
+    :param code: the status code to exit with.
+    """
+    def __init__(self, code=0):
+        self.exit_code = code

python/click/formatting.py

@@ -0,0 +1,256 @@
+from contextlib import contextmanager
+from .termui import get_terminal_size
+from .parser import split_opt
+from ._compat import term_len
+
+
+# Can force a width.  This is used by the test system
+FORCED_WIDTH = None
+
+
+def measure_table(rows):
+    widths = {}
+    for row in rows:
+        for idx, col in enumerate(row):
+            widths[idx] = max(widths.get(idx, 0), term_len(col))
+    return tuple(y for x, y in sorted(widths.items()))
+
+
+def iter_rows(rows, col_count):
+    for row in rows:
+        row = tuple(row)
+        yield row + ('',) * (col_count - len(row))
+
+
+def wrap_text(text, width=78, initial_indent='', subsequent_indent='',
+              preserve_paragraphs=False):
+    """A helper function that intelligently wraps text.  By default, it
+    assumes that it operates on a single paragraph of text but if the
+    `preserve_paragraphs` parameter is provided it will intelligently
+    handle paragraphs (defined by two empty lines).
+
+    If paragraphs are handled, a paragraph can be prefixed with an empty
+    line containing the ``\\b`` character (``\\x08``) to indicate that
+    no rewrapping should happen in that block.
+
+    :param text: the text that should be rewrapped.
+    :param width: the maximum width for the text.
+    :param initial_indent: the initial indent that should be placed on the
+                           first line as a string.
+    :param subsequent_indent: the indent string that should be placed on
+                              each consecutive line.
+    :param preserve_paragraphs: if this flag is set then the wrapping will
+                                intelligently handle paragraphs.
+    """
+    from ._textwrap import TextWrapper
+    text = text.expandtabs()
+    wrapper = TextWrapper(width, initial_indent=initial_indent,
+                          subsequent_indent=subsequent_indent,
+                          replace_whitespace=False)
+    if not preserve_paragraphs:
+        return wrapper.fill(text)
+
+    p = []
+    buf = []
+    indent = None
+
+    def _flush_par():
+        if not buf:
+            return
+        if buf[0].strip() == '\b':
+            p.append((indent or 0, True, '\n'.join(buf[1:])))
+        else:
+            p.append((indent or 0, False, ' '.join(buf)))
+        del buf[:]
+
+    for line in text.splitlines():
+        if not line:
+            _flush_par()
+            indent = None
+        else:
+            if indent is None:
+                orig_len = term_len(line)
+                line = line.lstrip()
+                indent = orig_len - term_len(line)
+            buf.append(line)
+    _flush_par()
+
+    rv = []
+    for indent, raw, text in p:
+        with wrapper.extra_indent(' ' * indent):
+            if raw:
+                rv.append(wrapper.indent_only(text))
+            else:
+                rv.append(wrapper.fill(text))
+
+    return '\n\n'.join(rv)
+
+
+class HelpFormatter(object):
+    """This class helps with formatting text-based help pages.  It's
+    usually just needed for very special internal cases, but it's also
+    exposed so that developers can write their own fancy outputs.
+
+    At present, it always writes into memory.
+
+    :param indent_increment: the additional increment for each level.
+    :param width: the width for the text.  This defaults to the terminal
+                  width clamped to a maximum of 78.
+    """
+
+    def __init__(self, indent_increment=2, width=None, max_width=None):
+        self.indent_increment = indent_increment
+        if max_width is None:
+            max_width = 80
+        if width is None:
+            width = FORCED_WIDTH
+            if width is None:
+                width = max(min(get_terminal_size()[0], max_width) - 2, 50)
+        self.width = width
+        self.current_indent = 0
+        self.buffer = []
+
+    def write(self, string):
+        """Writes a unicode string into the internal buffer."""
+        self.buffer.append(string)
+
+    def indent(self):
+        """Increases the indentation."""
+        self.current_indent += self.indent_increment
+
+    def dedent(self):
+        """Decreases the indentation."""
+        self.current_indent -= self.indent_increment
+
+    def write_usage(self, prog, args='', prefix='Usage: '):
+        """Writes a usage line into the buffer.
+
+        :param prog: the program name.
+        :param args: whitespace separated list of arguments.
+        :param prefix: the prefix for the first line.
+        """
+        usage_prefix = '%*s%s ' % (self.current_indent, prefix, prog)
+        text_width = self.width - self.current_indent
+
+        if text_width >= (term_len(usage_prefix) + 20):
+            # The arguments will fit to the right of the prefix.
+            indent = ' ' * term_len(usage_prefix)
+            self.write(wrap_text(args, text_width,
+                                 initial_indent=usage_prefix,
+                                 subsequent_indent=indent))
+        else:
+            # The prefix is too long, put the arguments on the next line.
+            self.write(usage_prefix)
+            self.write('\n')
+            indent = ' ' * (max(self.current_indent, term_len(prefix)) + 4)
+            self.write(wrap_text(args, text_width,
+                                 initial_indent=indent,
+                                 subsequent_indent=indent))
+
+        self.write('\n')
+
+    def write_heading(self, heading):
+        """Writes a heading into the buffer."""
+        self.write('%*s%s:\n' % (self.current_indent, '', heading))
+
+    def write_paragraph(self):
+        """Writes a paragraph into the buffer."""
+        if self.buffer:
+            self.write('\n')
+
+    def write_text(self, text):
+        """Writes re-indented text into the buffer.  This rewraps and
+        preserves paragraphs.
+        """
+        text_width = max(self.width - self.current_indent, 11)
+        indent = ' ' * self.current_indent
+        self.write(wrap_text(text, text_width,
+                             initial_indent=indent,
+                             subsequent_indent=indent,
+                             preserve_paragraphs=True))
+        self.write('\n')
+
+    def write_dl(self, rows, col_max=30, col_spacing=2):
+        """Writes a definition list into the buffer.  This is how options
+        and commands are usually formatted.
+
+        :param rows: a list of two item tuples for the terms and values.
+        :param col_max: the maximum width of the first column.
+        :param col_spacing: the number of spaces between the first and
+                            second column.
+        """
+        rows = list(rows)
+        widths = measure_table(rows)
+        if len(widths) != 2:
+            raise TypeError('Expected two columns for definition list')
+
+        first_col = min(widths[0], col_max) + col_spacing
+
+        for first, second in iter_rows(rows, len(widths)):
+            self.write('%*s%s' % (self.current_indent, '', first))
+            if not second:
+                self.write('\n')
+                continue
+            if term_len(first) <= first_col - col_spacing:
+                self.write(' ' * (first_col - term_len(first)))
+            else:
+                self.write('\n')
+                self.write(' ' * (first_col + self.current_indent))
+
+            text_width = max(self.width - first_col - 2, 10)
+            lines = iter(wrap_text(second, text_width).splitlines())
+            if lines:
+                self.write(next(lines) + '\n')
+                for line in lines:
+                    self.write('%*s%s\n' % (
+                        first_col + self.current_indent, '', line))
+            else:
+                self.write('\n')
+
+    @contextmanager
+    def section(self, name):
+        """Helpful context manager that writes a paragraph, a heading,
+        and the indents.
+
+        :param name: the section name that is written as heading.
+        """
+        self.write_paragraph()
+        self.write_heading(name)
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    @contextmanager
+    def indentation(self):
+        """A context manager that increases the indentation."""
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    def getvalue(self):
+        """Returns the buffer contents."""
+        return ''.join(self.buffer)
+
+
+def join_options(options):
+    """Given a list of option strings this joins them in the most appropriate
+    way and returns them in the form ``(formatted_string,
+    any_prefix_is_slash)`` where the second item in the tuple is a flag that
+    indicates if any of the option prefixes was a slash.
+    """
+    rv = []
+    any_prefix_is_slash = False
+    for opt in options:
+        prefix = split_opt(opt)[0]
+        if prefix == '/':
+            any_prefix_is_slash = True
+        rv.append((len(prefix), opt))
+
+    rv.sort(key=lambda x: x[0])
+
+    rv = ', '.join(x[1] for x in rv)
+    return rv, any_prefix_is_slash

python/click/globals.py

@@ -0,0 +1,48 @@
+from threading import local
+
+
+_local = local()
+
+
+def get_current_context(silent=False):
+    """Returns the current click context.  This can be used as a way to
+    access the current context object from anywhere.  This is a more implicit
+    alternative to the :func:`pass_context` decorator.  This function is
+    primarily useful for helpers such as :func:`echo` which might be
+    interested in changing its behavior based on the current context.
+
+    To push the current context, :meth:`Context.scope` can be used.
+
+    .. versionadded:: 5.0
+
+    :param silent: is set to `True` the return value is `None` if no context
+                   is available.  The default behavior is to raise a
+                   :exc:`RuntimeError`.
+    """
+    try:
+        return getattr(_local, 'stack')[-1]
+    except (AttributeError, IndexError):
+        if not silent:
+            raise RuntimeError('There is no active click context.')
+
+
+def push_context(ctx):
+    """Pushes a new context to the current stack."""
+    _local.__dict__.setdefault('stack', []).append(ctx)
+
+
+def pop_context():
+    """Removes the top level from the stack."""
+    _local.stack.pop()
+
+
+def resolve_color_default(color=None):
+    """"Internal helper to get the default value of the color flag.  If a
+    value is passed it's returned unchanged, otherwise it's looked up from
+    the current context.
+    """
+    if color is not None:
+        return color
+    ctx = get_current_context(silent=True)
+    if ctx is not None:
+        return ctx.color

python/click/parser.py

@@ -0,0 +1,427 @@
+# -*- coding: utf-8 -*-
+"""
+click.parser
+~~~~~~~~~~~~
+
+This module started out as largely a copy paste from the stdlib's
+optparse module with the features removed that we do not need from
+optparse because we implement them in Click on a higher level (for
+instance type handling, help formatting and a lot more).
+
+The plan is to remove more and more from here over time.
+
+The reason this is a different module and not optparse from the stdlib
+is that there are differences in 2.x and 3.x about the error messages
+generated and optparse in the stdlib uses gettext for no good reason
+and might cause us issues.
+"""
+
+import re
+from collections import deque
+from .exceptions import UsageError, NoSuchOption, BadOptionUsage, \
+     BadArgumentUsage
+
+
+def _unpack_args(args, nargs_spec):
+    """Given an iterable of arguments and an iterable of nargs specifications,
+    it returns a tuple with all the unpacked arguments at the first index
+    and all remaining arguments as the second.
+
+    The nargs specification is the number of arguments that should be consumed
+    or `-1` to indicate that this position should eat up all the remainders.
+
+    Missing items are filled with `None`.
+    """
+    args = deque(args)
+    nargs_spec = deque(nargs_spec)
+    rv = []
+    spos = None
+
+    def _fetch(c):
+        try:
+            if spos is None:
+                return c.popleft()
+            else:
+                return c.pop()
+        except IndexError:
+            return None
+
+    while nargs_spec:
+        nargs = _fetch(nargs_spec)
+        if nargs == 1:
+            rv.append(_fetch(args))
+        elif nargs > 1:
+            x = [_fetch(args) for _ in range(nargs)]
+            # If we're reversed, we're pulling in the arguments in reverse,
+            # so we need to turn them around.
+            if spos is not None:
+                x.reverse()
+            rv.append(tuple(x))
+        elif nargs < 0:
+            if spos is not None:
+                raise TypeError('Cannot have two nargs < 0')
+            spos = len(rv)
+            rv.append(None)
+
+    # spos is the position of the wildcard (star).  If it's not `None`,
+    # we fill it with the remainder.
+    if spos is not None:
+        rv[spos] = tuple(args)
+        args = []
+        rv[spos + 1:] = reversed(rv[spos + 1:])
+
+    return tuple(rv), list(args)
+
+
+def _error_opt_args(nargs, opt):
+    if nargs == 1:
+        raise BadOptionUsage(opt, '%s option requires an argument' % opt)
+    raise BadOptionUsage(opt, '%s option requires %d arguments' % (opt, nargs))
+
+
+def split_opt(opt):
+    first = opt[:1]
+    if first.isalnum():
+        return '', opt
+    if opt[1:2] == first:
+        return opt[:2], opt[2:]
+    return first, opt[1:]
+
+
+def normalize_opt(opt, ctx):
+    if ctx is None or ctx.token_normalize_func is None:
+        return opt
+    prefix, opt = split_opt(opt)
+    return prefix + ctx.token_normalize_func(opt)
+
+
+def split_arg_string(string):
+    """Given an argument string this attempts to split it into small parts."""
+    rv = []
+    for match in re.finditer(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+                             r'|"([^"\\]*(?:\\.[^"\\]*)*)"'
+                             r'|\S+)\s*', string, re.S):
+        arg = match.group().strip()
+        if arg[:1] == arg[-1:] and arg[:1] in '"\'':
+            arg = arg[1:-1].encode('ascii', 'backslashreplace') \
+                .decode('unicode-escape')
+        try:
+            arg = type(string)(arg)
+        except UnicodeError:
+            pass
+        rv.append(arg)
+    return rv
+
+
+class Option(object):
+
+    def __init__(self, opts, dest, action=None, nargs=1, const=None, obj=None):
+        self._short_opts = []
+        self._long_opts = []
+        self.prefixes = set()
+
+        for opt in opts:
+            prefix, value = split_opt(opt)
+            if not prefix:
+                raise ValueError('Invalid start character for option (%s)'
+                                 % opt)
+            self.prefixes.add(prefix[0])
+            if len(prefix) == 1 and len(value) == 1:
+                self._short_opts.append(opt)
+            else:
+                self._long_opts.append(opt)
+                self.prefixes.add(prefix)
+
+        if action is None:
+            action = 'store'
+
+        self.dest = dest
+        self.action = action
+        self.nargs = nargs
+        self.const = const
+        self.obj = obj
+
+    @property
+    def takes_value(self):
+        return self.action in ('store', 'append')
+
+    def process(self, value, state):
+        if self.action == 'store':
+            state.opts[self.dest] = value
+        elif self.action == 'store_const':
+            state.opts[self.dest] = self.const
+        elif self.action == 'append':
+            state.opts.setdefault(self.dest, []).append(value)
+        elif self.action == 'append_const':
+            state.opts.setdefault(self.dest, []).append(self.const)
+        elif self.action == 'count':
+            state.opts[self.dest] = state.opts.get(self.dest, 0) + 1
+        else:
+            raise ValueError('unknown action %r' % self.action)
+        state.order.append(self.obj)
+
+
+class Argument(object):
+
+    def __init__(self, dest, nargs=1, obj=None):
+        self.dest = dest
+        self.nargs = nargs
+        self.obj = obj
+
+    def process(self, value, state):
+        if self.nargs > 1:
+            holes = sum(1 for x in value if x is None)
+            if holes == len(value):
+                value = None
+            elif holes != 0:
+                raise BadArgumentUsage('argument %s takes %d values'
+                                       % (self.dest, self.nargs))
+        state.opts[self.dest] = value
+        state.order.append(self.obj)
+
+
+class ParsingState(object):
+
+    def __init__(self, rargs):
+        self.opts = {}
+        self.largs = []
+        self.rargs = rargs
+        self.order = []
+
+
+class OptionParser(object):
+    """The option parser is an internal class that is ultimately used to
+    parse options and arguments.  It's modelled after optparse and brings
+    a similar but vastly simplified API.  It should generally not be used
+    directly as the high level Click classes wrap it for you.
+
+    It's not nearly as extensible as optparse or argparse as it does not
+    implement features that are implemented on a higher level (such as
+    types or defaults).
+
+    :param ctx: optionally the :class:`~click.Context` where this parser
+                should go with.
+    """
+
+    def __init__(self, ctx=None):
+        #: The :class:`~click.Context` for this parser.  This might be
+        #: `None` for some advanced use cases.
+        self.ctx = ctx
+        #: This controls how the parser deals with interspersed arguments.
+        #: If this is set to `False`, the parser will stop on the first
+        #: non-option.  Click uses this to implement nested subcommands
+        #: safely.
+        self.allow_interspersed_args = True
+        #: This tells the parser how to deal with unknown options.  By
+        #: default it will error out (which is sensible), but there is a
+        #: second mode where it will ignore it and continue processing
+        #: after shifting all the unknown options into the resulting args.
+        self.ignore_unknown_options = False
+        if ctx is not None:
+            self.allow_interspersed_args = ctx.allow_interspersed_args
+            self.ignore_unknown_options = ctx.ignore_unknown_options
+        self._short_opt = {}
+        self._long_opt = {}
+        self._opt_prefixes = set(['-', '--'])
+        self._args = []
+
+    def add_option(self, opts, dest, action=None, nargs=1, const=None,
+                   obj=None):
+        """Adds a new option named `dest` to the parser.  The destination
+        is not inferred (unlike with optparse) and needs to be explicitly
+        provided.  Action can be any of ``store``, ``store_const``,
+        ``append``, ``appnd_const`` or ``count``.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        opts = [normalize_opt(opt, self.ctx) for opt in opts]
+        option = Option(opts, dest, action=action, nargs=nargs,
+                        const=const, obj=obj)
+        self._opt_prefixes.update(option.prefixes)
+        for opt in option._short_opts:
+            self._short_opt[opt] = option
+        for opt in option._long_opts:
+            self._long_opt[opt] = option
+
+    def add_argument(self, dest, nargs=1, obj=None):
+        """Adds a positional argument named `dest` to the parser.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        self._args.append(Argument(dest=dest, nargs=nargs, obj=obj))
+
+    def parse_args(self, args):
+        """Parses positional arguments and returns ``(values, args, order)``
+        for the parsed options and arguments as well as the leftover
+        arguments if there are any.  The order is a list of objects as they
+        appear on the command line.  If arguments appear multiple times they
+        will be memorized multiple times as well.
+        """
+        state = ParsingState(args)
+        try:
+            self._process_args_for_options(state)
+            self._process_args_for_args(state)
+        except UsageError:
+            if self.ctx is None or not self.ctx.resilient_parsing:
+                raise
+        return state.opts, state.largs, state.order
+
+    def _process_args_for_args(self, state):
+        pargs, args = _unpack_args(state.largs + state.rargs,
+                                   [x.nargs for x in self._args])
+
+        for idx, arg in enumerate(self._args):
+            arg.process(pargs[idx], state)
+
+        state.largs = args
+        state.rargs = []
+
+    def _process_args_for_options(self, state):
+        while state.rargs:
+            arg = state.rargs.pop(0)
+            arglen = len(arg)
+            # Double dashes always handled explicitly regardless of what
+            # prefixes are valid.
+            if arg == '--':
+                return
+            elif arg[:1] in self._opt_prefixes and arglen > 1:
+                self._process_opts(arg, state)
+            elif self.allow_interspersed_args:
+                state.largs.append(arg)
+            else:
+                state.rargs.insert(0, arg)
+                return
+
+        # Say this is the original argument list:
+        # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)]
+        #                            ^
+        # (we are about to process arg(i)).
+        #
+        # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of
+        # [arg0, ..., arg(i-1)] (any options and their arguments will have
+        # been removed from largs).
+        #
+        # The while loop will usually consume 1 or more arguments per pass.
+        # If it consumes 1 (eg. arg is an option that takes no arguments),
+        # then after _process_arg() is done the situation is:
+        #
+        #   largs = subset of [arg0, ..., arg(i)]
+        #   rargs = [arg(i+1), ..., arg(N-1)]
+        #
+        # If allow_interspersed_args is false, largs will always be
+        # *empty* -- still a subset of [arg0, ..., arg(i-1)], but
+        # not a very interesting subset!
+
+    def _match_long_opt(self, opt, explicit_value, state):
+        if opt not in self._long_opt:
+            possibilities = [word for word in self._long_opt
+                             if word.startswith(opt)]
+            raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx)
+
+        option = self._long_opt[opt]
+        if option.takes_value:
+            # At this point it's safe to modify rargs by injecting the
+            # explicit value, because no exception is raised in this
+            # branch.  This means that the inserted value will be fully
+            # consumed.
+            if explicit_value is not None:
+                state.rargs.insert(0, explicit_value)
+
+            nargs = option.nargs
+            if len(state.rargs) < nargs:
+                _error_opt_args(nargs, opt)
+            elif nargs == 1:
+                value = state.rargs.pop(0)
+            else:
+                value = tuple(state.rargs[:nargs])
+                del state.rargs[:nargs]
+
+        elif explicit_value is not None:
+            raise BadOptionUsage(opt, '%s option does not take a value' % opt)
+
+        else:
+            value = None
+
+        option.process(value, state)
+
+    def _match_short_opt(self, arg, state):
+        stop = False
+        i = 1
+        prefix = arg[0]
+        unknown_options = []
+
+        for ch in arg[1:]:
+            opt = normalize_opt(prefix + ch, self.ctx)
+            option = self._short_opt.get(opt)
+            i += 1
+
+            if not option:
+                if self.ignore_unknown_options:
+                    unknown_options.append(ch)
+                    continue
+                raise NoSuchOption(opt, ctx=self.ctx)
+            if option.takes_value:
+                # Any characters left in arg?  Pretend they're the
+                # next arg, and stop consuming characters of arg.
+                if i < len(arg):
+                    state.rargs.insert(0, arg[i:])
+                    stop = True
+
+                nargs = option.nargs
+                if len(state.rargs) < nargs:
+                    _error_opt_args(nargs, opt)
+                elif nargs == 1:
+                    value = state.rargs.pop(0)
+                else:
+                    value = tuple(state.rargs[:nargs])
+                    del state.rargs[:nargs]
+
+            else:
+                value = None
+
+            option.process(value, state)
+
+            if stop:
+                break
+
+        # If we got any unknown options we re-combinate the string of the
+        # remaining options and re-attach the prefix, then report that
+        # to the state as new larg.  This way there is basic combinatorics
+        # that can be achieved while still ignoring unknown arguments.
+        if self.ignore_unknown_options and unknown_options:
+            state.largs.append(prefix + ''.join(unknown_options))
+
+    def _process_opts(self, arg, state):
+        explicit_value = None
+        # Long option handling happens in two parts.  The first part is
+        # supporting explicitly attached values.  In any case, we will try
+        # to long match the option first.
+        if '=' in arg:
+            long_opt, explicit_value = arg.split('=', 1)
+        else:
+            long_opt = arg
+        norm_long_opt = normalize_opt(long_opt, self.ctx)
+
+        # At this point we will match the (assumed) long option through
+        # the long option matching code.  Note that this allows options
+        # like "-foo" to be matched as long options.
+        try:
+            self._match_long_opt(norm_long_opt, explicit_value, state)
+        except NoSuchOption:
+            # At this point the long option matching failed, and we need
+            # to try with short options.  However there is a special rule
+            # which says, that if we have a two character options prefix
+            # (applies to "--foo" for instance), we do not dispatch to the
+            # short option code and will instead raise the no option
+            # error.
+            if arg[:2] not in self._opt_prefixes:
+                return self._match_short_opt(arg, state)
+            if not self.ignore_unknown_options:
+                raise
+            state.largs.append(arg)

python/click/termui.py

@@ -0,0 +1,606 @@
+import os
+import sys
+import struct
+import inspect
+import itertools
+
+from ._compat import raw_input, text_type, string_types, \
+     isatty, strip_ansi, get_winterm_size, DEFAULT_COLUMNS, WIN
+from .utils import echo
+from .exceptions import Abort, UsageError
+from .types import convert_type, Choice, Path
+from .globals import resolve_color_default
+
+
+# The prompt functions to use.  The doc tools currently override these
+# functions to customize how they work.
+visible_prompt_func = raw_input
+
+_ansi_colors = {
+    'black': 30,
+    'red': 31,
+    'green': 32,
+    'yellow': 33,
+    'blue': 34,
+    'magenta': 35,
+    'cyan': 36,
+    'white': 37,
+    'reset': 39,
+    'bright_black': 90,
+    'bright_red': 91,
+    'bright_green': 92,
+    'bright_yellow': 93,
+    'bright_blue': 94,
+    'bright_magenta': 95,
+    'bright_cyan': 96,
+    'bright_white': 97,
+}
+_ansi_reset_all = '\033[0m'
+
+
+def hidden_prompt_func(prompt):
+    import getpass
+    return getpass.getpass(prompt)
+
+
+def _build_prompt(text, suffix, show_default=False, default=None, show_choices=True, type=None):
+    prompt = text
+    if type is not None and show_choices and isinstance(type, Choice):
+        prompt += ' (' + ", ".join(map(str, type.choices)) + ')'
+    if default is not None and show_default:
+        prompt = '%s [%s]' % (prompt, default)
+    return prompt + suffix
+
+
+def prompt(text, default=None, hide_input=False, confirmation_prompt=False,
+           type=None, value_proc=None, prompt_suffix=': ', show_default=True,
+           err=False, show_choices=True):
+    """Prompts a user for input.  This is a convenience function that can
+    be used to prompt a user for input later.
+
+    If the user aborts the input by sending a interrupt signal, this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 7.0
+       Added the show_choices parameter.
+
+    .. versionadded:: 6.0
+       Added unicode support for cmd.exe on Windows.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the text to show for the prompt.
+    :param default: the default value to use if no input happens.  If this
+                    is not given it will prompt until it's aborted.
+    :param hide_input: if this is set to true then the input value will
+                       be hidden.
+    :param confirmation_prompt: asks for confirmation for the value.
+    :param type: the type to use to check the value against.
+    :param value_proc: if this parameter is provided it's a function that
+                       is invoked instead of the type conversion to
+                       convert a value.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    :param show_choices: Show or hide choices if the passed type is a Choice.
+                         For example if type is a Choice of either day or week,
+                         show_choices is true and text is "Group by" then the
+                         prompt will be "Group by (day, week): ".
+    """
+    result = None
+
+    def prompt_func(text):
+        f = hide_input and hidden_prompt_func or visible_prompt_func
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(text, nl=False, err=err)
+            return f('')
+        except (KeyboardInterrupt, EOFError):
+            # getpass doesn't print a newline if the user aborts input with ^C.
+            # Allegedly this behavior is inherited from getpass(3).
+            # A doc bug has been filed at https://bugs.python.org/issue24711
+            if hide_input:
+                echo(None, err=err)
+            raise Abort()
+
+    if value_proc is None:
+        value_proc = convert_type(type, default)
+
+    prompt = _build_prompt(text, prompt_suffix, show_default, default, show_choices, type)
+
+    while 1:
+        while 1:
+            value = prompt_func(prompt)
+            if value:
+                break
+            elif default is not None:
+                if isinstance(value_proc, Path):
+                    # validate Path default value(exists, dir_okay etc.)
+                    value = default
+                    break
+                return default
+        try:
+            result = value_proc(value)
+        except UsageError as e:
+            echo('Error: %s' % e.message, err=err)
+            continue
+        if not confirmation_prompt:
+            return result
+        while 1:
+            value2 = prompt_func('Repeat for confirmation: ')
+            if value2:
+                break
+        if value == value2:
+            return result
+        echo('Error: the two entered values do not match', err=err)
+
+
+def confirm(text, default=False, abort=False, prompt_suffix=': ',
+            show_default=True, err=False):
+    """Prompts for confirmation (yes/no question).
+
+    If the user aborts the input by sending a interrupt signal this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the question to ask.
+    :param default: the default for the prompt.
+    :param abort: if this is set to `True` a negative answer aborts the
+                  exception by raising :exc:`Abort`.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    prompt = _build_prompt(text, prompt_suffix, show_default,
+                           default and 'Y/n' or 'y/N')
+    while 1:
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(prompt, nl=False, err=err)
+            value = visible_prompt_func('').lower().strip()
+        except (KeyboardInterrupt, EOFError):
+            raise Abort()
+        if value in ('y', 'yes'):
+            rv = True
+        elif value in ('n', 'no'):
+            rv = False
+        elif value == '':
+            rv = default
+        else:
+            echo('Error: invalid input', err=err)
+            continue
+        break
+    if abort and not rv:
+        raise Abort()
+    return rv
+
+
+def get_terminal_size():
+    """Returns the current size of the terminal as tuple in the form
+    ``(width, height)`` in columns and rows.
+    """
+    # If shutil has get_terminal_size() (Python 3.3 and later) use that
+    if sys.version_info >= (3, 3):
+        import shutil
+        shutil_get_terminal_size = getattr(shutil, 'get_terminal_size', None)
+        if shutil_get_terminal_size:
+            sz = shutil_get_terminal_size()
+            return sz.columns, sz.lines
+
+    # We provide a sensible default for get_winterm_size() when being invoked
+    # inside a subprocess. Without this, it would not provide a useful input.
+    if get_winterm_size is not None:
+        size = get_winterm_size()
+        if size == (0, 0):
+            return (79, 24)
+        else:
+            return size
+
+    def ioctl_gwinsz(fd):
+        try:
+            import fcntl
+            import termios
+            cr = struct.unpack(
+                'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
+        except Exception:
+            return
+        return cr
+
+    cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
+    if not cr:
+        try:
+            fd = os.open(os.ctermid(), os.O_RDONLY)
+            try:
+                cr = ioctl_gwinsz(fd)
+            finally:
+                os.close(fd)
+        except Exception:
+            pass
+    if not cr or not cr[0] or not cr[1]:
+        cr = (os.environ.get('LINES', 25),
+              os.environ.get('COLUMNS', DEFAULT_COLUMNS))
+    return int(cr[1]), int(cr[0])
+
+
+def echo_via_pager(text_or_generator, color=None):
+    """This function takes a text and shows it via an environment specific
+    pager on stdout.
+
+    .. versionchanged:: 3.0
+       Added the `color` flag.
+
+    :param text_or_generator: the text to page, or alternatively, a
+                              generator emitting the text to page.
+    :param color: controls if the pager supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    color = resolve_color_default(color)
+
+    if inspect.isgeneratorfunction(text_or_generator):
+        i = text_or_generator()
+    elif isinstance(text_or_generator, string_types):
+        i = [text_or_generator]
+    else:
+        i = iter(text_or_generator)
+
+    # convert every element of i to a text type if necessary
+    text_generator = (el if isinstance(el, string_types) else text_type(el)
+                      for el in i)
+
+    from ._termui_impl import pager
+    return pager(itertools.chain(text_generator, "\n"), color)
+
+
+def progressbar(iterable=None, length=None, label=None, show_eta=True,
+                show_percent=None, show_pos=False,
+                item_show_func=None, fill_char='#', empty_char='-',
+                bar_template='%(label)s  [%(bar)s]  %(info)s',
+                info_sep='  ', width=36, file=None, color=None):
+    """This function creates an iterable context manager that can be used
+    to iterate over something while showing a progress bar.  It will
+    either iterate over the `iterable` or `length` items (that are counted
+    up).  While iteration happens, this function will print a rendered
+    progress bar to the given `file` (defaults to stdout) and will attempt
+    to calculate remaining time and more.  By default, this progress bar
+    will not be rendered if the file is not a terminal.
+
+    The context manager creates the progress bar.  When the context
+    manager is entered the progress bar is already displayed.  With every
+    iteration over the progress bar, the iterable passed to the bar is
+    advanced and the bar is updated.  When the context manager exits,
+    a newline is printed and the progress bar is finalized on screen.
+
+    No printing must happen or the progress bar will be unintentionally
+    destroyed.
+
+    Example usage::
+
+        with progressbar(items) as bar:
+            for item in bar:
+                do_something_with(item)
+
+    Alternatively, if no iterable is specified, one can manually update the
+    progress bar through the `update()` method instead of directly
+    iterating over the progress bar.  The update method accepts the number
+    of steps to increment the bar with::
+
+        with progressbar(length=chunks.total_bytes) as bar:
+            for chunk in chunks:
+                process_chunk(chunk)
+                bar.update(chunks.bytes)
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `color` parameter.  Added a `update` method to the
+       progressbar object.
+
+    :param iterable: an iterable to iterate over.  If not provided the length
+                     is required.
+    :param length: the number of items to iterate over.  By default the
+                   progressbar will attempt to ask the iterator about its
+                   length, which might or might not work.  If an iterable is
+                   also provided this parameter can be used to override the
+                   length.  If an iterable is not provided the progress bar
+                   will iterate over a range of that length.
+    :param label: the label to show next to the progress bar.
+    :param show_eta: enables or disables the estimated time display.  This is
+                     automatically disabled if the length cannot be
+                     determined.
+    :param show_percent: enables or disables the percentage display.  The
+                         default is `True` if the iterable has a length or
+                         `False` if not.
+    :param show_pos: enables or disables the absolute position display.  The
+                     default is `False`.
+    :param item_show_func: a function called with the current item which
+                           can return a string to show the current item
+                           next to the progress bar.  Note that the current
+                           item can be `None`!
+    :param fill_char: the character to use to show the filled part of the
+                      progress bar.
+    :param empty_char: the character to use to show the non-filled part of
+                       the progress bar.
+    :param bar_template: the format string to use as template for the bar.
+                         The parameters in it are ``label`` for the label,
+                         ``bar`` for the progress bar and ``info`` for the
+                         info section.
+    :param info_sep: the separator between multiple info items (eta etc.)
+    :param width: the width of the progress bar in characters, 0 means full
+                  terminal width
+    :param file: the file to write to.  If this is not a terminal then
+                 only the label is printed.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.  This is only needed if ANSI
+                  codes are included anywhere in the progress bar output
+                  which is not the case by default.
+    """
+    from ._termui_impl import ProgressBar
+    color = resolve_color_default(color)
+    return ProgressBar(iterable=iterable, length=length, show_eta=show_eta,
+                       show_percent=show_percent, show_pos=show_pos,
+                       item_show_func=item_show_func, fill_char=fill_char,
+                       empty_char=empty_char, bar_template=bar_template,
+                       info_sep=info_sep, file=file, label=label,
+                       width=width, color=color)
+
+
+def clear():
+    """Clears the terminal screen.  This will have the effect of clearing
+    the whole visible space of the terminal and moving the cursor to the
+    top left.  This does not do anything if not connected to a terminal.
+
+    .. versionadded:: 2.0
+    """
+    if not isatty(sys.stdout):
+        return
+    # If we're on Windows and we don't have colorama available, then we
+    # clear the screen by shelling out.  Otherwise we can use an escape
+    # sequence.
+    if WIN:
+        os.system('cls')
+    else:
+        sys.stdout.write('\033[2J\033[1;1H')
+
+
+def style(text, fg=None, bg=None, bold=None, dim=None, underline=None,
+          blink=None, reverse=None, reset=True):
+    """Styles a text with ANSI styles and returns the new string.  By
+    default the styling is self contained which means that at the end
+    of the string a reset code is issued.  This can be prevented by
+    passing ``reset=False``.
+
+    Examples::
+
+        click.echo(click.style('Hello World!', fg='green'))
+        click.echo(click.style('ATTENTION!', blink=True))
+        click.echo(click.style('Some things', reverse=True, fg='cyan'))
+
+    Supported color names:
+
+    * ``black`` (might be a gray)
+    * ``red``
+    * ``green``
+    * ``yellow`` (might be an orange)
+    * ``blue``
+    * ``magenta``
+    * ``cyan``
+    * ``white`` (might be light gray)
+    * ``bright_black``
+    * ``bright_red``
+    * ``bright_green``
+    * ``bright_yellow``
+    * ``bright_blue``
+    * ``bright_magenta``
+    * ``bright_cyan``
+    * ``bright_white``
+    * ``reset`` (reset the color code only)
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 7.0
+       Added support for bright colors.
+
+    :param text: the string to style with ansi codes.
+    :param fg: if provided this will become the foreground color.
+    :param bg: if provided this will become the background color.
+    :param bold: if provided this will enable or disable bold mode.
+    :param dim: if provided this will enable or disable dim mode.  This is
+                badly supported.
+    :param underline: if provided this will enable or disable underline.
+    :param blink: if provided this will enable or disable blinking.
+    :param reverse: if provided this will enable or disable inverse
+                    rendering (foreground becomes background and the
+                    other way round).
+    :param reset: by default a reset-all code is added at the end of the
+                  string which means that styles do not carry over.  This
+                  can be disabled to compose styles.
+    """
+    bits = []
+    if fg:
+        try:
+            bits.append('\033[%dm' % (_ansi_colors[fg]))
+        except KeyError:
+            raise TypeError('Unknown color %r' % fg)
+    if bg:
+        try:
+            bits.append('\033[%dm' % (_ansi_colors[bg] + 10))
+        except KeyError:
+            raise TypeError('Unknown color %r' % bg)
+    if bold is not None:
+        bits.append('\033[%dm' % (1 if bold else 22))
+    if dim is not None:
+        bits.append('\033[%dm' % (2 if dim else 22))
+    if underline is not None:
+        bits.append('\033[%dm' % (4 if underline else 24))
+    if blink is not None:
+        bits.append('\033[%dm' % (5 if blink else 25))
+    if reverse is not None:
+        bits.append('\033[%dm' % (7 if reverse else 27))
+    bits.append(text)
+    if reset:
+        bits.append(_ansi_reset_all)
+    return ''.join(bits)
+
+
+def unstyle(text):
+    """Removes ANSI styling information from a string.  Usually it's not
+    necessary to use this function as Click's echo function will
+    automatically remove styling if necessary.
+
+    .. versionadded:: 2.0
+
+    :param text: the text to remove style information from.
+    """
+    return strip_ansi(text)
+
+
+def secho(message=None, file=None, nl=True, err=False, color=None, **styles):
+    """This function combines :func:`echo` and :func:`style` into one
+    call.  As such the following two calls are the same::
+
+        click.secho('Hello World!', fg='green')
+        click.echo(click.style('Hello World!', fg='green'))
+
+    All keyword arguments are forwarded to the underlying functions
+    depending on which one they go with.
+
+    .. versionadded:: 2.0
+    """
+    if message is not None:
+        message = style(message, **styles)
+    return echo(message, file=file, nl=nl, err=err, color=color)
+
+
+def edit(text=None, editor=None, env=None, require_save=True,
+         extension='.txt', filename=None):
+    r"""Edits the given text in the defined editor.  If an editor is given
+    (should be the full path to the executable but the regular operating
+    system search path is used for finding the executable) it overrides
+    the detected editor.  Optionally, some environment variables can be
+    used.  If the editor is closed without changes, `None` is returned.  In
+    case a file is edited directly the return value is always `None` and
+    `require_save` and `extension` are ignored.
+
+    If the editor cannot be opened a :exc:`UsageError` is raised.
+
+    Note for Windows: to simplify cross-platform usage, the newlines are
+    automatically converted from POSIX to Windows and vice versa.  As such,
+    the message here will have ``\n`` as newline markers.
+
+    :param text: the text to edit.
+    :param editor: optionally the editor to use.  Defaults to automatic
+                   detection.
+    :param env: environment variables to forward to the editor.
+    :param require_save: if this is true, then not saving in the editor
+                         will make the return value become `None`.
+    :param extension: the extension to tell the editor about.  This defaults
+                      to `.txt` but changing this might change syntax
+                      highlighting.
+    :param filename: if provided it will edit this file instead of the
+                     provided text contents.  It will not use a temporary
+                     file as an indirection in that case.
+    """
+    from ._termui_impl import Editor
+    editor = Editor(editor=editor, env=env, require_save=require_save,
+                    extension=extension)
+    if filename is None:
+        return editor.edit(text)
+    editor.edit_file(filename)
+
+
+def launch(url, wait=False, locate=False):
+    """This function launches the given URL (or filename) in the default
+    viewer application for this file type.  If this is an executable, it
+    might launch the executable in a new session.  The return value is
+    the exit code of the launched application.  Usually, ``0`` indicates
+    success.
+
+    Examples::
+
+        click.launch('https://click.palletsprojects.com/')
+        click.launch('/my/downloaded/file', locate=True)
+
+    .. versionadded:: 2.0
+
+    :param url: URL or filename of the thing to launch.
+    :param wait: waits for the program to stop.
+    :param locate: if this is set to `True` then instead of launching the
+                   application associated with the URL it will attempt to
+                   launch a file manager with the file located.  This
+                   might have weird effects if the URL does not point to
+                   the filesystem.
+    """
+    from ._termui_impl import open_url
+    return open_url(url, wait=wait, locate=locate)
+
+
+# If this is provided, getchar() calls into this instead.  This is used
+# for unittesting purposes.
+_getchar = None
+
+
+def getchar(echo=False):
+    """Fetches a single character from the terminal and returns it.  This
+    will always return a unicode character and under certain rare
+    circumstances this might return more than one character.  The
+    situations which more than one character is returned is when for
+    whatever reason multiple characters end up in the terminal buffer or
+    standard input was not actually a terminal.
+
+    Note that this will always read from the terminal, even if something
+    is piped into the standard input.
+
+    Note for Windows: in rare cases when typing non-ASCII characters, this
+    function might wait for a second character and then return both at once.
+    This is because certain Unicode characters look like special-key markers.
+
+    .. versionadded:: 2.0
+
+    :param echo: if set to `True`, the character read will also show up on
+                 the terminal.  The default is to not show it.
+    """
+    f = _getchar
+    if f is None:
+        from ._termui_impl import getchar as f
+    return f(echo)
+
+
+def raw_terminal():
+    from ._termui_impl import raw_terminal as f
+    return f()
+
+
+def pause(info='Press any key to continue ...', err=False):
+    """This command stops execution and waits for the user to press any
+    key to continue.  This is similar to the Windows batch "pause"
+    command.  If the program is not run through a terminal, this command
+    will instead do nothing.
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param info: the info string to print before pausing.
+    :param err: if set to message goes to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    if not isatty(sys.stdin) or not isatty(sys.stdout):
+        return
+    try:
+        if info:
+            echo(info, nl=False, err=err)
+        try:
+            getchar()
+        except (KeyboardInterrupt, EOFError):
+            pass
+    finally:
+        if info:
+            echo(err=err)

python/click/testing.py

@@ -0,0 +1,374 @@
+import os
+import sys
+import shutil
+import tempfile
+import contextlib
+import shlex
+
+from ._compat import iteritems, PY2, string_types
+
+
+# If someone wants to vendor click, we want to ensure the
+# correct package is discovered.  Ideally we could use a
+# relative import here but unfortunately Python does not
+# support that.
+clickpkg = sys.modules[__name__.rsplit('.', 1)[0]]
+
+
+if PY2:
+    from cStringIO import StringIO
+else:
+    import io
+    from ._compat import _find_binary_reader
+
+
+class EchoingStdin(object):
+
+    def __init__(self, input, output):
+        self._input = input
+        self._output = output
+
+    def __getattr__(self, x):
+        return getattr(self._input, x)
+
+    def _echo(self, rv):
+        self._output.write(rv)
+        return rv
+
+    def read(self, n=-1):
+        return self._echo(self._input.read(n))
+
+    def readline(self, n=-1):
+        return self._echo(self._input.readline(n))
+
+    def readlines(self):
+        return [self._echo(x) for x in self._input.readlines()]
+
+    def __iter__(self):
+        return iter(self._echo(x) for x in self._input)
+
+    def __repr__(self):
+        return repr(self._input)
+
+
+def make_input_stream(input, charset):
+    # Is already an input stream.
+    if hasattr(input, 'read'):
+        if PY2:
+            return input
+        rv = _find_binary_reader(input)
+        if rv is not None:
+            return rv
+        raise TypeError('Could not find binary reader for input stream.')
+
+    if input is None:
+        input = b''
+    elif not isinstance(input, bytes):
+        input = input.encode(charset)
+    if PY2:
+        return StringIO(input)
+    return io.BytesIO(input)
+
+
+class Result(object):
+    """Holds the captured result of an invoked CLI script."""
+
+    def __init__(self, runner, stdout_bytes, stderr_bytes, exit_code,
+                 exception, exc_info=None):
+        #: The runner that created the result
+        self.runner = runner
+        #: The standard output as bytes.
+        self.stdout_bytes = stdout_bytes
+        #: The standard error as bytes, or False(y) if not available
+        self.stderr_bytes = stderr_bytes
+        #: The exit code as integer.
+        self.exit_code = exit_code
+        #: The exception that happened if one did.
+        self.exception = exception
+        #: The traceback
+        self.exc_info = exc_info
+
+    @property
+    def output(self):
+        """The (standard) output as unicode string."""
+        return self.stdout
+
+    @property
+    def stdout(self):
+        """The standard output as unicode string."""
+        return self.stdout_bytes.decode(self.runner.charset, 'replace') \
+            .replace('\r\n', '\n')
+
+    @property
+    def stderr(self):
+        """The standard error as unicode string."""
+        if not self.stderr_bytes:
+            raise ValueError("stderr not separately captured")
+        return self.stderr_bytes.decode(self.runner.charset, 'replace') \
+            .replace('\r\n', '\n')
+
+
+    def __repr__(self):
+        return '<%s %s>' % (
+            type(self).__name__,
+            self.exception and repr(self.exception) or 'okay',
+        )
+
+
+class CliRunner(object):
+    """The CLI runner provides functionality to invoke a Click command line
+    script for unittesting purposes in a isolated environment.  This only
+    works in single-threaded systems without any concurrency as it changes the
+    global interpreter state.
+
+    :param charset: the character set for the input and output data.  This is
+                    UTF-8 by default and should not be changed currently as
+                    the reporting to Click only works in Python 2 properly.
+    :param env: a dictionary with environment variables for overriding.
+    :param echo_stdin: if this is set to `True`, then reading from stdin writes
+                       to stdout.  This is useful for showing examples in
+                       some circumstances.  Note that regular prompts
+                       will automatically echo the input.
+    :param mix_stderr: if this is set to `False`, then stdout and stderr are
+                       preserved as independent streams.  This is useful for
+                       Unix-philosophy apps that have predictable stdout and
+                       noisy stderr, such that each may be measured
+                       independently
+    """
+
+    def __init__(self, charset=None, env=None, echo_stdin=False,
+                 mix_stderr=True):
+        if charset is None:
+            charset = 'utf-8'
+        self.charset = charset
+        self.env = env or {}
+        self.echo_stdin = echo_stdin
+        self.mix_stderr = mix_stderr
+
+    def get_default_prog_name(self, cli):
+        """Given a command object it will return the default program name
+        for it.  The default is the `name` attribute or ``"root"`` if not
+        set.
+        """
+        return cli.name or 'root'
+
+    def make_env(self, overrides=None):
+        """Returns the environment overrides for invoking a script."""
+        rv = dict(self.env)
+        if overrides:
+            rv.update(overrides)
+        return rv
+
+    @contextlib.contextmanager
+    def isolation(self, input=None, env=None, color=False):
+        """A context manager that sets up the isolation for invoking of a
+        command line tool.  This sets up stdin with the given input data
+        and `os.environ` with the overrides from the given dictionary.
+        This also rebinds some internals in Click to be mocked (like the
+        prompt functionality).
+
+        This is automatically done in the :meth:`invoke` method.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param input: the input stream to put into sys.stdin.
+        :param env: the environment overrides as dictionary.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        input = make_input_stream(input, self.charset)
+
+        old_stdin = sys.stdin
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        old_forced_width = clickpkg.formatting.FORCED_WIDTH
+        clickpkg.formatting.FORCED_WIDTH = 80
+
+        env = self.make_env(env)
+
+        if PY2:
+            bytes_output = StringIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+            sys.stdout = bytes_output
+            if not self.mix_stderr:
+                bytes_error = StringIO()
+                sys.stderr = bytes_error
+        else:
+            bytes_output = io.BytesIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+            input = io.TextIOWrapper(input, encoding=self.charset)
+            sys.stdout = io.TextIOWrapper(
+                bytes_output, encoding=self.charset)
+            if not self.mix_stderr:
+                bytes_error = io.BytesIO()
+                sys.stderr = io.TextIOWrapper(
+                    bytes_error, encoding=self.charset)
+
+        if self.mix_stderr:
+            sys.stderr = sys.stdout
+
+        sys.stdin = input
+
+        def visible_input(prompt=None):
+            sys.stdout.write(prompt or '')
+            val = input.readline().rstrip('\r\n')
+            sys.stdout.write(val + '\n')
+            sys.stdout.flush()
+            return val
+
+        def hidden_input(prompt=None):
+            sys.stdout.write((prompt or '') + '\n')
+            sys.stdout.flush()
+            return input.readline().rstrip('\r\n')
+
+        def _getchar(echo):
+            char = sys.stdin.read(1)
+            if echo:
+                sys.stdout.write(char)
+                sys.stdout.flush()
+            return char
+
+        default_color = color
+
+        def should_strip_ansi(stream=None, color=None):
+            if color is None:
+                return not default_color
+            return not color
+
+        old_visible_prompt_func = clickpkg.termui.visible_prompt_func
+        old_hidden_prompt_func = clickpkg.termui.hidden_prompt_func
+        old__getchar_func = clickpkg.termui._getchar
+        old_should_strip_ansi = clickpkg.utils.should_strip_ansi
+        clickpkg.termui.visible_prompt_func = visible_input
+        clickpkg.termui.hidden_prompt_func = hidden_input
+        clickpkg.termui._getchar = _getchar
+        clickpkg.utils.should_strip_ansi = should_strip_ansi
+
+        old_env = {}
+        try:
+            for key, value in iteritems(env):
+                old_env[key] = os.environ.get(key)
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            yield (bytes_output, not self.mix_stderr and bytes_error)
+        finally:
+            for key, value in iteritems(old_env):
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+            sys.stdin = old_stdin
+            clickpkg.termui.visible_prompt_func = old_visible_prompt_func
+            clickpkg.termui.hidden_prompt_func = old_hidden_prompt_func
+            clickpkg.termui._getchar = old__getchar_func
+            clickpkg.utils.should_strip_ansi = old_should_strip_ansi
+            clickpkg.formatting.FORCED_WIDTH = old_forced_width
+
+    def invoke(self, cli, args=None, input=None, env=None,
+               catch_exceptions=True, color=False, mix_stderr=False, **extra):
+        """Invokes a command in an isolated environment.  The arguments are
+        forwarded directly to the command line script, the `extra` keyword
+        arguments are passed to the :meth:`~clickpkg.Command.main` function of
+        the command.
+
+        This returns a :class:`Result` object.
+
+        .. versionadded:: 3.0
+           The ``catch_exceptions`` parameter was added.
+
+        .. versionchanged:: 3.0
+           The result object now has an `exc_info` attribute with the
+           traceback if available.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param cli: the command to invoke
+        :param args: the arguments to invoke. It may be given as an iterable
+                     or a string. When given as string it will be interpreted
+                     as a Unix shell command. More details at
+                     :func:`shlex.split`.
+        :param input: the input data for `sys.stdin`.
+        :param env: the environment overrides.
+        :param catch_exceptions: Whether to catch any other exceptions than
+                                 ``SystemExit``.
+        :param extra: the keyword arguments to pass to :meth:`main`.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        exc_info = None
+        with self.isolation(input=input, env=env, color=color) as outstreams:
+            exception = None
+            exit_code = 0
+
+            if isinstance(args, string_types):
+                args = shlex.split(args)
+
+            try:
+                prog_name = extra.pop("prog_name")
+            except KeyError:
+                prog_name = self.get_default_prog_name(cli)
+
+            try:
+                cli.main(args=args or (), prog_name=prog_name, **extra)
+            except SystemExit as e:
+                exc_info = sys.exc_info()
+                exit_code = e.code
+                if exit_code is None:
+                    exit_code = 0
+
+                if exit_code != 0:
+                    exception = e
+
+                if not isinstance(exit_code, int):
+                    sys.stdout.write(str(exit_code))
+                    sys.stdout.write('\n')
+                    exit_code = 1
+
+            except Exception as e:
+                if not catch_exceptions:
+                    raise
+                exception = e
+                exit_code = 1
+                exc_info = sys.exc_info()
+            finally:
+                sys.stdout.flush()
+                stdout = outstreams[0].getvalue()
+                stderr = outstreams[1] and outstreams[1].getvalue()
+
+        return Result(runner=self,
+                      stdout_bytes=stdout,
+                      stderr_bytes=stderr,
+                      exit_code=exit_code,
+                      exception=exception,
+                      exc_info=exc_info)
+
+    @contextlib.contextmanager
+    def isolated_filesystem(self):
+        """A context manager that creates a temporary folder and changes
+        the current working directory to it for isolated filesystem tests.
+        """
+        cwd = os.getcwd()
+        t = tempfile.mkdtemp()
+        os.chdir(t)
+        try:
+            yield t
+        finally:
+            os.chdir(cwd)
+            try:
+                shutil.rmtree(t)
+            except (OSError, IOError):
+                pass

python/click/types.py

@@ -0,0 +1,668 @@
+import os
+import stat
+from datetime import datetime
+
+from ._compat import open_stream, text_type, filename_to_ui, \
+    get_filesystem_encoding, get_streerror, _get_argv_encoding, PY2
+from .exceptions import BadParameter
+from .utils import safecall, LazyFile
+
+
+class ParamType(object):
+    """Helper for converting values through types.  The following is
+    necessary for a valid type:
+
+    *   it needs a name
+    *   it needs to pass through None unchanged
+    *   it needs to convert from a string
+    *   it needs to convert its result type through unchanged
+        (eg: needs to be idempotent)
+    *   it needs to be able to deal with param and context being `None`.
+        This can be the case when the object is used with prompt
+        inputs.
+    """
+    is_composite = False
+
+    #: the descriptive name of this type
+    name = None
+
+    #: if a list of this type is expected and the value is pulled from a
+    #: string environment variable, this is what splits it up.  `None`
+    #: means any whitespace.  For all parameters the general rule is that
+    #: whitespace splits them up.  The exception are paths and files which
+    #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on
+    #: Windows).
+    envvar_list_splitter = None
+
+    def __call__(self, value, param=None, ctx=None):
+        if value is not None:
+            return self.convert(value, param, ctx)
+
+    def get_metavar(self, param):
+        """Returns the metavar default for this param if it provides one."""
+
+    def get_missing_message(self, param):
+        """Optionally might return extra information about a missing
+        parameter.
+
+        .. versionadded:: 2.0
+        """
+
+    def convert(self, value, param, ctx):
+        """Converts the value.  This is not invoked for values that are
+        `None` (the missing value).
+        """
+        return value
+
+    def split_envvar_value(self, rv):
+        """Given a value from an environment variable this splits it up
+        into small chunks depending on the defined envvar list splitter.
+
+        If the splitter is set to `None`, which means that whitespace splits,
+        then leading and trailing whitespace is ignored.  Otherwise, leading
+        and trailing splitters usually lead to empty items being included.
+        """
+        return (rv or '').split(self.envvar_list_splitter)
+
+    def fail(self, message, param=None, ctx=None):
+        """Helper method to fail with an invalid value message."""
+        raise BadParameter(message, ctx=ctx, param=param)
+
+
+class CompositeParamType(ParamType):
+    is_composite = True
+
+    @property
+    def arity(self):
+        raise NotImplementedError()
+
+
+class FuncParamType(ParamType):
+
+    def __init__(self, func):
+        self.name = func.__name__
+        self.func = func
+
+    def convert(self, value, param, ctx):
+        try:
+            return self.func(value)
+        except ValueError:
+            try:
+                value = text_type(value)
+            except UnicodeError:
+                value = str(value).decode('utf-8', 'replace')
+            self.fail(value, param, ctx)
+
+
+class UnprocessedParamType(ParamType):
+    name = 'text'
+
+    def convert(self, value, param, ctx):
+        return value
+
+    def __repr__(self):
+        return 'UNPROCESSED'
+
+
+class StringParamType(ParamType):
+    name = 'text'
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bytes):
+            enc = _get_argv_encoding()
+            try:
+                value = value.decode(enc)
+            except UnicodeError:
+                fs_enc = get_filesystem_encoding()
+                if fs_enc != enc:
+                    try:
+                        value = value.decode(fs_enc)
+                    except UnicodeError:
+                        value = value.decode('utf-8', 'replace')
+            return value
+        return value
+
+    def __repr__(self):
+        return 'STRING'
+
+
+class Choice(ParamType):
+    """The choice type allows a value to be checked against a fixed set
+    of supported values. All of these values have to be strings.
+
+    You should only pass a list or tuple of choices. Other iterables
+    (like generators) may lead to surprising results.
+
+    See :ref:`choice-opts` for an example.
+
+    :param case_sensitive: Set to false to make choices case
+        insensitive. Defaults to true.
+    """
+
+    name = 'choice'
+
+    def __init__(self, choices, case_sensitive=True):
+        self.choices = choices
+        self.case_sensitive = case_sensitive
+
+    def get_metavar(self, param):
+        return '[%s]' % '|'.join(self.choices)
+
+    def get_missing_message(self, param):
+        return 'Choose from:\n\t%s.' % ',\n\t'.join(self.choices)
+
+    def convert(self, value, param, ctx):
+        # Exact match
+        if value in self.choices:
+            return value
+
+        # Match through normalization and case sensitivity
+        # first do token_normalize_func, then lowercase
+        # preserve original `value` to produce an accurate message in
+        # `self.fail`
+        normed_value = value
+        normed_choices = self.choices
+
+        if ctx is not None and \
+           ctx.token_normalize_func is not None:
+            normed_value = ctx.token_normalize_func(value)
+            normed_choices = [ctx.token_normalize_func(choice) for choice in
+                              self.choices]
+
+        if not self.case_sensitive:
+            normed_value = normed_value.lower()
+            normed_choices = [choice.lower() for choice in normed_choices]
+
+        if normed_value in normed_choices:
+            return normed_value
+
+        self.fail('invalid choice: %s. (choose from %s)' %
+                  (value, ', '.join(self.choices)), param, ctx)
+
+    def __repr__(self):
+        return 'Choice(%r)' % list(self.choices)
+
+
+class DateTime(ParamType):
+    """The DateTime type converts date strings into `datetime` objects.
+
+    The format strings which are checked are configurable, but default to some
+    common (non-timezone aware) ISO 8601 formats.
+
+    When specifying *DateTime* formats, you should only pass a list or a tuple.
+    Other iterables, like generators, may lead to surprising results.
+
+    The format strings are processed using ``datetime.strptime``, and this
+    consequently defines the format strings which are allowed.
+
+    Parsing is tried using each format, in order, and the first format which
+    parses successfully is used.
+
+    :param formats: A list or tuple of date format strings, in the order in
+                    which they should be tried. Defaults to
+                    ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``,
+                    ``'%Y-%m-%d %H:%M:%S'``.
+    """
+    name = 'datetime'
+
+    def __init__(self, formats=None):
+        self.formats = formats or [
+            '%Y-%m-%d',
+            '%Y-%m-%dT%H:%M:%S',
+            '%Y-%m-%d %H:%M:%S'
+        ]
+
+    def get_metavar(self, param):
+        return '[{}]'.format('|'.join(self.formats))
+
+    def _try_to_convert_date(self, value, format):
+        try:
+            return datetime.strptime(value, format)
+        except ValueError:
+            return None
+
+    def convert(self, value, param, ctx):
+        # Exact match
+        for format in self.formats:
+            dtime = self._try_to_convert_date(value, format)
+            if dtime:
+                return dtime
+
+        self.fail(
+            'invalid datetime format: {}. (choose from {})'.format(
+                value, ', '.join(self.formats)))
+
+    def __repr__(self):
+        return 'DateTime'
+
+
+class IntParamType(ParamType):
+    name = 'integer'
+
+    def convert(self, value, param, ctx):
+        try:
+            return int(value)
+        except (ValueError, UnicodeError):
+            self.fail('%s is not a valid integer' % value, param, ctx)
+
+    def __repr__(self):
+        return 'INT'
+
+
+class IntRange(IntParamType):
+    """A parameter that works similar to :data:`click.INT` but restricts
+    the value to fit into a range.  The default behavior is to fail if the
+    value falls outside the range, but it can also be silently clamped
+    between the two edges.
+
+    See :ref:`ranges` for an example.
+    """
+    name = 'integer range'
+
+    def __init__(self, min=None, max=None, clamp=False):
+        self.min = min
+        self.max = max
+        self.clamp = clamp
+
+    def convert(self, value, param, ctx):
+        rv = IntParamType.convert(self, value, param, ctx)
+        if self.clamp:
+            if self.min is not None and rv < self.min:
+                return self.min
+            if self.max is not None and rv > self.max:
+                return self.max
+        if self.min is not None and rv < self.min or \
+           self.max is not None and rv > self.max:
+            if self.min is None:
+                self.fail('%s is bigger than the maximum valid value '
+                          '%s.' % (rv, self.max), param, ctx)
+            elif self.max is None:
+                self.fail('%s is smaller than the minimum valid value '
+                          '%s.' % (rv, self.min), param, ctx)
+            else:
+                self.fail('%s is not in the valid range of %s to %s.'
+                          % (rv, self.min, self.max), param, ctx)
+        return rv
+
+    def __repr__(self):
+        return 'IntRange(%r, %r)' % (self.min, self.max)
+
+
+class FloatParamType(ParamType):
+    name = 'float'
+
+    def convert(self, value, param, ctx):
+        try:
+            return float(value)
+        except (UnicodeError, ValueError):
+            self.fail('%s is not a valid floating point value' %
+                      value, param, ctx)
+
+    def __repr__(self):
+        return 'FLOAT'
+
+
+class FloatRange(FloatParamType):
+    """A parameter that works similar to :data:`click.FLOAT` but restricts
+    the value to fit into a range.  The default behavior is to fail if the
+    value falls outside the range, but it can also be silently clamped
+    between the two edges.
+
+    See :ref:`ranges` for an example.
+    """
+    name = 'float range'
+
+    def __init__(self, min=None, max=None, clamp=False):
+        self.min = min
+        self.max = max
+        self.clamp = clamp
+
+    def convert(self, value, param, ctx):
+        rv = FloatParamType.convert(self, value, param, ctx)
+        if self.clamp:
+            if self.min is not None and rv < self.min:
+                return self.min
+            if self.max is not None and rv > self.max:
+                return self.max
+        if self.min is not None and rv < self.min or \
+           self.max is not None and rv > self.max:
+            if self.min is None:
+                self.fail('%s is bigger than the maximum valid value '
+                          '%s.' % (rv, self.max), param, ctx)
+            elif self.max is None:
+                self.fail('%s is smaller than the minimum valid value '
+                          '%s.' % (rv, self.min), param, ctx)
+            else:
+                self.fail('%s is not in the valid range of %s to %s.'
+                          % (rv, self.min, self.max), param, ctx)
+        return rv
+
+    def __repr__(self):
+        return 'FloatRange(%r, %r)' % (self.min, self.max)
+
+
+class BoolParamType(ParamType):
+    name = 'boolean'
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bool):
+            return bool(value)
+        value = value.lower()
+        if value in ('true', 't', '1', 'yes', 'y'):
+            return True
+        elif value in ('false', 'f', '0', 'no', 'n'):
+            return False
+        self.fail('%s is not a valid boolean' % value, param, ctx)
+
+    def __repr__(self):
+        return 'BOOL'
+
+
+class UUIDParameterType(ParamType):
+    name = 'uuid'
+
+    def convert(self, value, param, ctx):
+        import uuid
+        try:
+            if PY2 and isinstance(value, text_type):
+                value = value.encode('ascii')
+            return uuid.UUID(value)
+        except (UnicodeError, ValueError):
+            self.fail('%s is not a valid UUID value' % value, param, ctx)
+
+    def __repr__(self):
+        return 'UUID'
+
+
+class File(ParamType):
+    """Declares a parameter to be a file for reading or writing.  The file
+    is automatically closed once the context tears down (after the command
+    finished working).
+
+    Files can be opened for reading or writing.  The special value ``-``
+    indicates stdin or stdout depending on the mode.
+
+    By default, the file is opened for reading text data, but it can also be
+    opened in binary mode or for writing.  The encoding parameter can be used
+    to force a specific encoding.
+
+    The `lazy` flag controls if the file should be opened immediately or upon
+    first IO. The default is to be non-lazy for standard input and output
+    streams as well as files opened for reading, `lazy` otherwise. When opening a
+    file lazily for reading, it is still opened temporarily for validation, but
+    will not be held open until first IO. lazy is mainly useful when opening
+    for writing to avoid creating the file until it is needed.
+
+    Starting with Click 2.0, files can also be opened atomically in which
+    case all writes go into a separate file in the same folder and upon
+    completion the file will be moved over to the original location.  This
+    is useful if a file regularly read by other users is modified.
+
+    See :ref:`file-args` for more information.
+    """
+    name = 'filename'
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(self, mode='r', encoding=None, errors='strict', lazy=None,
+                 atomic=False):
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.lazy = lazy
+        self.atomic = atomic
+
+    def resolve_lazy_flag(self, value):
+        if self.lazy is not None:
+            return self.lazy
+        if value == '-':
+            return False
+        elif 'w' in self.mode:
+            return True
+        return False
+
+    def convert(self, value, param, ctx):
+        try:
+            if hasattr(value, 'read') or hasattr(value, 'write'):
+                return value
+
+            lazy = self.resolve_lazy_flag(value)
+
+            if lazy:
+                f = LazyFile(value, self.mode, self.encoding, self.errors,
+                             atomic=self.atomic)
+                if ctx is not None:
+                    ctx.call_on_close(f.close_intelligently)
+                return f
+
+            f, should_close = open_stream(value, self.mode,
+                                          self.encoding, self.errors,
+                                          atomic=self.atomic)
+            # If a context is provided, we automatically close the file
+            # at the end of the context execution (or flush out).  If a
+            # context does not exist, it's the caller's responsibility to
+            # properly close the file.  This for instance happens when the
+            # type is used with prompts.
+            if ctx is not None:
+                if should_close:
+                    ctx.call_on_close(safecall(f.close))
+                else:
+                    ctx.call_on_close(safecall(f.flush))
+            return f
+        except (IOError, OSError) as e:
+            self.fail('Could not open file: %s: %s' % (
+                filename_to_ui(value),
+                get_streerror(e),
+            ), param, ctx)
+
+
+class Path(ParamType):
+    """The path type is similar to the :class:`File` type but it performs
+    different checks.  First of all, instead of returning an open file
+    handle it returns just the filename.  Secondly, it can perform various
+    basic checks about what the file or directory should be.
+
+    .. versionchanged:: 6.0
+       `allow_dash` was added.
+
+    :param exists: if set to true, the file or directory needs to exist for
+                   this value to be valid.  If this is not required and a
+                   file does indeed not exist, then all further checks are
+                   silently skipped.
+    :param file_okay: controls if a file is a possible value.
+    :param dir_okay: controls if a directory is a possible value.
+    :param writable: if true, a writable check is performed.
+    :param readable: if true, a readable check is performed.
+    :param resolve_path: if this is true, then the path is fully resolved
+                         before the value is passed onwards.  This means
+                         that it's absolute and symlinks are resolved.  It
+                         will not expand a tilde-prefix, as this is
+                         supposed to be done by the shell only.
+    :param allow_dash: If this is set to `True`, a single dash to indicate
+                       standard streams is permitted.
+    :param path_type: optionally a string type that should be used to
+                      represent the path.  The default is `None` which
+                      means the return value will be either bytes or
+                      unicode depending on what makes most sense given the
+                      input data Click deals with.
+    """
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(self, exists=False, file_okay=True, dir_okay=True,
+                 writable=False, readable=True, resolve_path=False,
+                 allow_dash=False, path_type=None):
+        self.exists = exists
+        self.file_okay = file_okay
+        self.dir_okay = dir_okay
+        self.writable = writable
+        self.readable = readable
+        self.resolve_path = resolve_path
+        self.allow_dash = allow_dash
+        self.type = path_type
+
+        if self.file_okay and not self.dir_okay:
+            self.name = 'file'
+            self.path_type = 'File'
+        elif self.dir_okay and not self.file_okay:
+            self.name = 'directory'
+            self.path_type = 'Directory'
+        else:
+            self.name = 'path'
+            self.path_type = 'Path'
+
+    def coerce_path_result(self, rv):
+        if self.type is not None and not isinstance(rv, self.type):
+            if self.type is text_type:
+                rv = rv.decode(get_filesystem_encoding())
+            else:
+                rv = rv.encode(get_filesystem_encoding())
+        return rv
+
+    def convert(self, value, param, ctx):
+        rv = value
+
+        is_dash = self.file_okay and self.allow_dash and rv in (b'-', '-')
+
+        if not is_dash:
+            if self.resolve_path:
+                rv = os.path.realpath(rv)
+
+            try:
+                st = os.stat(rv)
+            except OSError:
+                if not self.exists:
+                    return self.coerce_path_result(rv)
+                self.fail('%s "%s" does not exist.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+
+            if not self.file_okay and stat.S_ISREG(st.st_mode):
+                self.fail('%s "%s" is a file.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if not self.dir_okay and stat.S_ISDIR(st.st_mode):
+                self.fail('%s "%s" is a directory.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if self.writable and not os.access(value, os.W_OK):
+                self.fail('%s "%s" is not writable.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if self.readable and not os.access(value, os.R_OK):
+                self.fail('%s "%s" is not readable.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+
+        return self.coerce_path_result(rv)
+
+
+class Tuple(CompositeParamType):
+    """The default behavior of Click is to apply a type on a value directly.
+    This works well in most cases, except for when `nargs` is set to a fixed
+    count and different types should be used for different items.  In this
+    case the :class:`Tuple` type can be used.  This type can only be used
+    if `nargs` is set to a fixed number.
+
+    For more information see :ref:`tuple-type`.
+
+    This can be selected by using a Python tuple literal as a type.
+
+    :param types: a list of types that should be used for the tuple items.
+    """
+
+    def __init__(self, types):
+        self.types = [convert_type(ty) for ty in types]
+
+    @property
+    def name(self):
+        return "<" + " ".join(ty.name for ty in self.types) + ">"
+
+    @property
+    def arity(self):
+        return len(self.types)
+
+    def convert(self, value, param, ctx):
+        if len(value) != len(self.types):
+            raise TypeError('It would appear that nargs is set to conflict '
+                            'with the composite type arity.')
+        return tuple(ty(x, param, ctx) for ty, x in zip(self.types, value))
+
+
+def convert_type(ty, default=None):
+    """Converts a callable or python ty into the most appropriate param
+    ty.
+    """
+    guessed_type = False
+    if ty is None and default is not None:
+        if isinstance(default, tuple):
+            ty = tuple(map(type, default))
+        else:
+            ty = type(default)
+        guessed_type = True
+
+    if isinstance(ty, tuple):
+        return Tuple(ty)
+    if isinstance(ty, ParamType):
+        return ty
+    if ty is text_type or ty is str or ty is None:
+        return STRING
+    if ty is int:
+        return INT
+    # Booleans are only okay if not guessed.  This is done because for
+    # flags the default value is actually a bit of a lie in that it
+    # indicates which of the flags is the one we want.  See get_default()
+    # for more information.
+    if ty is bool and not guessed_type:
+        return BOOL
+    if ty is float:
+        return FLOAT
+    if guessed_type:
+        return STRING
+
+    # Catch a common mistake
+    if __debug__:
+        try:
+            if issubclass(ty, ParamType):
+                raise AssertionError('Attempted to use an uninstantiated '
+                                     'parameter type (%s).' % ty)
+        except TypeError:
+            pass
+    return FuncParamType(ty)
+
+
+#: A dummy parameter type that just does nothing.  From a user's
+#: perspective this appears to just be the same as `STRING` but internally
+#: no string conversion takes place.  This is necessary to achieve the
+#: same bytes/unicode behavior on Python 2/3 in situations where you want
+#: to not convert argument types.  This is usually useful when working
+#: with file paths as they can appear in bytes and unicode.
+#:
+#: For path related uses the :class:`Path` type is a better choice but
+#: there are situations where an unprocessed type is useful which is why
+#: it is is provided.
+#:
+#: .. versionadded:: 4.0
+UNPROCESSED = UnprocessedParamType()
+
+#: A unicode string parameter type which is the implicit default.  This
+#: can also be selected by using ``str`` as type.
+STRING = StringParamType()
+
+#: An integer parameter.  This can also be selected by using ``int`` as
+#: type.
+INT = IntParamType()
+
+#: A floating point value parameter.  This can also be selected by using
+#: ``float`` as type.
+FLOAT = FloatParamType()
+
+#: A boolean parameter.  This is the default for boolean flags.  This can
+#: also be selected by using ``bool`` as a type.
+BOOL = BoolParamType()
+
+#: A UUID parameter.
+UUID = UUIDParameterType()

python/click/utils.py

@@ -0,0 +1,440 @@
+import os
+import sys
+
+from .globals import resolve_color_default
+
+from ._compat import text_type, open_stream, get_filesystem_encoding, \
+    get_streerror, string_types, PY2, binary_streams, text_streams, \
+    filename_to_ui, auto_wrap_for_ansi, strip_ansi, should_strip_ansi, \
+    _default_text_stdout, _default_text_stderr, is_bytes, WIN
+
+if not PY2:
+    from ._compat import _find_binary_writer
+elif WIN:
+    from ._winconsole import _get_windows_argv, \
+         _hash_py_argv, _initial_argv_hash
+
+
+echo_native_types = string_types + (bytes, bytearray)
+
+
+def _posixify(name):
+    return '-'.join(name.split()).lower()
+
+
+def safecall(func):
+    """Wraps a function so that it swallows exceptions."""
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            pass
+    return wrapper
+
+
+def make_str(value):
+    """Converts a value into a valid string."""
+    if isinstance(value, bytes):
+        try:
+            return value.decode(get_filesystem_encoding())
+        except UnicodeError:
+            return value.decode('utf-8', 'replace')
+    return text_type(value)
+
+
+def make_default_short_help(help, max_length=45):
+    """Return a condensed version of help string."""
+    words = help.split()
+    total_length = 0
+    result = []
+    done = False
+
+    for word in words:
+        if word[-1:] == '.':
+            done = True
+        new_length = result and 1 + len(word) or len(word)
+        if total_length + new_length > max_length:
+            result.append('...')
+            done = True
+        else:
+            if result:
+                result.append(' ')
+            result.append(word)
+        if done:
+            break
+        total_length += new_length
+
+    return ''.join(result)
+
+
+class LazyFile(object):
+    """A lazy file works like a regular file but it does not fully open
+    the file but it does perform some basic checks early to see if the
+    filename parameter does make sense.  This is useful for safely opening
+    files for writing.
+    """
+
+    def __init__(self, filename, mode='r', encoding=None, errors='strict',
+                 atomic=False):
+        self.name = filename
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.atomic = atomic
+
+        if filename == '-':
+            self._f, self.should_close = open_stream(filename, mode,
+                                                     encoding, errors)
+        else:
+            if 'r' in mode:
+                # Open and close the file in case we're opening it for
+                # reading so that we can catch at least some errors in
+                # some cases early.
+                open(filename, mode).close()
+            self._f = None
+            self.should_close = True
+
+    def __getattr__(self, name):
+        return getattr(self.open(), name)
+
+    def __repr__(self):
+        if self._f is not None:
+            return repr(self._f)
+        return '<unopened file %r %s>' % (self.name, self.mode)
+
+    def open(self):
+        """Opens the file if it's not yet open.  This call might fail with
+        a :exc:`FileError`.  Not handling this error will produce an error
+        that Click shows.
+        """
+        if self._f is not None:
+            return self._f
+        try:
+            rv, self.should_close = open_stream(self.name, self.mode,
+                                                self.encoding,
+                                                self.errors,
+                                                atomic=self.atomic)
+        except (IOError, OSError) as e:
+            from .exceptions import FileError
+            raise FileError(self.name, hint=get_streerror(e))
+        self._f = rv
+        return rv
+
+    def close(self):
+        """Closes the underlying file, no matter what."""
+        if self._f is not None:
+            self._f.close()
+
+    def close_intelligently(self):
+        """This function only closes the file if it was opened by the lazy
+        file wrapper.  For instance this will never close stdin.
+        """
+        if self.should_close:
+            self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close_intelligently()
+
+    def __iter__(self):
+        self.open()
+        return iter(self._f)
+
+
+class KeepOpenFile(object):
+
+    def __init__(self, file):
+        self._file = file
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        pass
+
+    def __repr__(self):
+        return repr(self._file)
+
+    def __iter__(self):
+        return iter(self._file)
+
+
+def echo(message=None, file=None, nl=True, err=False, color=None):
+    """Prints a message plus a newline to the given file or stdout.  On
+    first sight, this looks like the print function, but it has improved
+    support for handling Unicode and binary data that does not fail no
+    matter how badly configured the system is.
+
+    Primarily it means that you can print binary data as well as Unicode
+    data on both 2.x and 3.x to the given file in the most appropriate way
+    possible.  This is a very carefree function in that it will try its
+    best to not fail.  As of Click 6.0 this includes support for unicode
+    output on the Windows console.
+
+    In addition to that, if `colorama`_ is installed, the echo function will
+    also support clever handling of ANSI codes.  Essentially it will then
+    do the following:
+
+    -   add transparent handling of ANSI color codes on Windows.
+    -   hide ANSI codes automatically if the destination file is not a
+        terminal.
+
+    .. _colorama: https://pypi.org/project/colorama/
+
+    .. versionchanged:: 6.0
+       As of Click 6.0 the echo function will properly support unicode
+       output on the windows console.  Not that click does not modify
+       the interpreter in any way which means that `sys.stdout` or the
+       print statement or function will still not provide unicode support.
+
+    .. versionchanged:: 2.0
+       Starting with version 2.0 of Click, the echo function will work
+       with colorama if it's installed.
+
+    .. versionadded:: 3.0
+       The `err` parameter was added.
+
+    .. versionchanged:: 4.0
+       Added the `color` flag.
+
+    :param message: the message to print
+    :param file: the file to write to (defaults to ``stdout``)
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``.  This is faster and easier than calling
+                :func:`get_text_stderr` yourself.
+    :param nl: if set to `True` (the default) a newline is printed afterwards.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    if file is None:
+        if err:
+            file = _default_text_stderr()
+        else:
+            file = _default_text_stdout()
+
+    # Convert non bytes/text into the native string type.
+    if message is not None and not isinstance(message, echo_native_types):
+        message = text_type(message)
+
+    if nl:
+        message = message or u''
+        if isinstance(message, text_type):
+            message += u'\n'
+        else:
+            message += b'\n'
+
+    # If there is a message, and we're in Python 3, and the value looks
+    # like bytes, we manually need to find the binary stream and write the
+    # message in there.  This is done separately so that most stream
+    # types will work as you would expect.  Eg: you can write to StringIO
+    # for other cases.
+    if message and not PY2 and is_bytes(message):
+        binary_file = _find_binary_writer(file)
+        if binary_file is not None:
+            file.flush()
+            binary_file.write(message)
+            binary_file.flush()
+            return
+
+    # ANSI-style support.  If there is no message or we are dealing with
+    # bytes nothing is happening.  If we are connected to a file we want
+    # to strip colors.  If we are on windows we either wrap the stream
+    # to strip the color or we use the colorama support to translate the
+    # ansi codes to API calls.
+    if message and not is_bytes(message):
+        color = resolve_color_default(color)
+        if should_strip_ansi(file, color):
+            message = strip_ansi(message)
+        elif WIN:
+            if auto_wrap_for_ansi is not None:
+                file = auto_wrap_for_ansi(file)
+            elif not color:
+                message = strip_ansi(message)
+
+    if message:
+        file.write(message)
+    file.flush()
+
+
+def get_binary_stream(name):
+    """Returns a system stream for byte processing.  This essentially
+    returns the stream from the sys module with the given name but it
+    solves some compatibility issues between different Python versions.
+    Primarily this function is necessary for getting binary streams on
+    Python 3.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    """
+    opener = binary_streams.get(name)
+    if opener is None:
+        raise TypeError('Unknown standard stream %r' % name)
+    return opener()
+
+
+def get_text_stream(name, encoding=None, errors='strict'):
+    """Returns a system stream for text processing.  This usually returns
+    a wrapped stream around a binary stream returned from
+    :func:`get_binary_stream` but it also can take shortcuts on Python 3
+    for already correctly configured streams.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    :param encoding: overrides the detected default encoding.
+    :param errors: overrides the default error mode.
+    """
+    opener = text_streams.get(name)
+    if opener is None:
+        raise TypeError('Unknown standard stream %r' % name)
+    return opener(encoding, errors)
+
+
+def open_file(filename, mode='r', encoding=None, errors='strict',
+              lazy=False, atomic=False):
+    """This is similar to how the :class:`File` works but for manual
+    usage.  Files are opened non lazy by default.  This can open regular
+    files as well as stdin/stdout if ``'-'`` is passed.
+
+    If stdin/stdout is returned the stream is wrapped so that the context
+    manager will not close the stream accidentally.  This makes it possible
+    to always use the function like this without having to worry to
+    accidentally close a standard stream::
+
+        with open_file(filename) as f:
+            ...
+
+    .. versionadded:: 3.0
+
+    :param filename: the name of the file to open (or ``'-'`` for stdin/stdout).
+    :param mode: the mode in which to open the file.
+    :param encoding: the encoding to use.
+    :param errors: the error handling for this file.
+    :param lazy: can be flipped to true to open the file lazily.
+    :param atomic: in atomic mode writes go into a temporary file and it's
+                   moved on close.
+    """
+    if lazy:
+        return LazyFile(filename, mode, encoding, errors, atomic=atomic)
+    f, should_close = open_stream(filename, mode, encoding, errors,
+                                  atomic=atomic)
+    if not should_close:
+        f = KeepOpenFile(f)
+    return f
+
+
+def get_os_args():
+    """This returns the argument part of sys.argv in the most appropriate
+    form for processing.  What this means is that this return value is in
+    a format that works for Click to process but does not necessarily
+    correspond well to what's actually standard for the interpreter.
+
+    On most environments the return value is ``sys.argv[:1]`` unchanged.
+    However if you are on Windows and running Python 2 the return value
+    will actually be a list of unicode strings instead because the
+    default behavior on that platform otherwise will not be able to
+    carry all possible values that sys.argv can have.
+
+    .. versionadded:: 6.0
+    """
+    # We can only extract the unicode argv if sys.argv has not been
+    # changed since the startup of the application.
+    if PY2 and WIN and _initial_argv_hash == _hash_py_argv():
+        return _get_windows_argv()
+    return sys.argv[1:]
+
+
+def format_filename(filename, shorten=False):
+    """Formats a filename for user display.  The main purpose of this
+    function is to ensure that the filename can be displayed at all.  This
+    will decode the filename to unicode if necessary in a way that it will
+    not fail.  Optionally, it can shorten the filename to not include the
+    full path to the filename.
+
+    :param filename: formats a filename for UI display.  This will also convert
+                     the filename into unicode without failing.
+    :param shorten: this optionally shortens the filename to strip of the
+                    path that leads up to it.
+    """
+    if shorten:
+        filename = os.path.basename(filename)
+    return filename_to_ui(filename)
+
+
+def get_app_dir(app_name, roaming=True, force_posix=False):
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Win XP (roaming):
+      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo Bar``
+    Win XP (not roaming):
+      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
+    Win 7 (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Win 7 (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no affect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+    if WIN:
+        key = roaming and 'APPDATA' or 'LOCALAPPDATA'
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser('~')
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(os.path.expanduser('~/.' + _posixify(app_name)))
+    if sys.platform == 'darwin':
+        return os.path.join(os.path.expanduser(
+            '~/Library/Application Support'), app_name)
+    return os.path.join(
+        os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')),
+        _posixify(app_name))
+
+
+class PacifyFlushWrapper(object):
+    """This wrapper is used to catch and suppress BrokenPipeErrors resulting
+    from ``.flush()`` being called on broken pipe during the shutdown/final-GC
+    of the Python interpreter. Notably ``.flush()`` is always called on
+    ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any
+    other cleanup code, and the case where the underlying file is not a broken
+    pipe, all calls and attributes are proxied.
+    """
+
+    def __init__(self, wrapped):
+        self.wrapped = wrapped
+
+    def flush(self):
+        try:
+            self.wrapped.flush()
+        except IOError as e:
+            import errno
+            if e.errno != errno.EPIPE:
+                raise
+
+    def __getattr__(self, attr):
+        return getattr(self.wrapped, attr)

python/flask/__init__.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+    flask
+    ~~~~~
+
+    A microframework based on Werkzeug.  It's extensively documented
+    and follows best practice patterns.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+__version__ = '1.0.3'
+
+# utilities we import from Werkzeug and Jinja2 that are unused
+# in the module but are exported as public interface.
+from werkzeug.exceptions import abort
+from werkzeug.utils import redirect
+from jinja2 import Markup, escape
+
+from .app import Flask, Request, Response
+from .config import Config
+from .helpers import url_for, flash, send_file, send_from_directory, \
+     get_flashed_messages, get_template_attribute, make_response, safe_join, \
+     stream_with_context
+from .globals import current_app, g, request, session, _request_ctx_stack, \
+     _app_ctx_stack
+from .ctx import has_request_context, has_app_context, \
+     after_this_request, copy_current_request_context
+from .blueprints import Blueprint
+from .templating import render_template, render_template_string
+
+# the signals
+from .signals import signals_available, template_rendered, request_started, \
+     request_finished, got_request_exception, request_tearing_down, \
+     appcontext_tearing_down, appcontext_pushed, \
+     appcontext_popped, message_flashed, before_render_template
+
+# We're not exposing the actual json module but a convenient wrapper around
+# it.
+from . import json
+
+# This was the only thing that Flask used to export at one point and it had
+# a more generic name.
+jsonify = json.jsonify
+
+# backwards compat, goes away in 1.0
+from .sessions import SecureCookieSession as Session
+json_available = True

python/flask/__main__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.__main__
+    ~~~~~~~~~~~~~~
+
+    Alias for flask.run for the command line.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+if __name__ == '__main__':
+    from .cli import main
+    main(as_module=True)

python/flask/_compat.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+    flask._compat
+    ~~~~~~~~~~~~~
+
+    Some py2/py3 compatibility support based on a stripped down
+    version of six so we don't have to depend on a specific version
+    of it.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import sys
+
+PY2 = sys.version_info[0] == 2
+_identity = lambda x: x
+
+
+if not PY2:
+    text_type = str
+    string_types = (str,)
+    integer_types = (int,)
+
+    iterkeys = lambda d: iter(d.keys())
+    itervalues = lambda d: iter(d.values())
+    iteritems = lambda d: iter(d.items())
+
+    from inspect import getfullargspec as getargspec
+    from io import StringIO
+    import collections.abc as collections_abc
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+    implements_to_string = _identity
+
+else:
+    text_type = unicode
+    string_types = (str, unicode)
+    integer_types = (int, long)
+
+    iterkeys = lambda d: d.iterkeys()
+    itervalues = lambda d: d.itervalues()
+    iteritems = lambda d: d.iteritems()
+
+    from inspect import getargspec
+    from cStringIO import StringIO
+    import collections as collections_abc
+
+    exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
+
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a
+    # dummy metaclass for one level of class instantiation that replaces
+    # itself with the actual metaclass.
+    class metaclass(type):
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+# Certain versions of pypy have a bug where clearing the exception stack
+# breaks the __exit__ function in a very peculiar way.  The second level of
+# exception blocks is necessary because pypy seems to forget to check if an
+# exception happened until the next bytecode instruction?
+#
+# Relevant PyPy bugfix commit:
+# https://bitbucket.org/pypy/pypy/commits/77ecf91c635a287e88e60d8ddb0f4e9df4003301
+# According to ronan on #pypy IRC, it is released in PyPy2 2.3 and later
+# versions.
+#
+# Ubuntu 14.04 has PyPy 2.2.1, which does exhibit this bug.
+BROKEN_PYPY_CTXMGR_EXIT = False
+if hasattr(sys, 'pypy_version_info'):
+    class _Mgr(object):
+        def __enter__(self):
+            return self
+        def __exit__(self, *args):
+            if hasattr(sys, 'exc_clear'):
+                # Python 3 (PyPy3) doesn't have exc_clear
+                sys.exc_clear()
+    try:
+        try:
+            with _Mgr():
+                raise AssertionError()
+        except:
+            raise
+    except TypeError:
+        BROKEN_PYPY_CTXMGR_EXIT = True
+    except AssertionError:
+        pass

python/flask/blueprints.py

@@ -0,0 +1,447 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.blueprints
+    ~~~~~~~~~~~~~~~~
+
+    Blueprints are the recommended way to implement larger or more
+    pluggable applications in Flask 0.7 and later.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+from functools import update_wrapper
+
+from .helpers import _PackageBoundObject, _endpoint_from_view_func
+
+
+class BlueprintSetupState(object):
+    """Temporary holder object for registering a blueprint with the
+    application.  An instance of this class is created by the
+    :meth:`~flask.Blueprint.make_setup_state` method and later passed
+    to all register callback functions.
+    """
+
+    def __init__(self, blueprint, app, options, first_registration):
+        #: a reference to the current application
+        self.app = app
+
+        #: a reference to the blueprint that created this setup state.
+        self.blueprint = blueprint
+
+        #: a dictionary with all options that were passed to the
+        #: :meth:`~flask.Flask.register_blueprint` method.
+        self.options = options
+
+        #: as blueprints can be registered multiple times with the
+        #: application and not everything wants to be registered
+        #: multiple times on it, this attribute can be used to figure
+        #: out if the blueprint was registered in the past already.
+        self.first_registration = first_registration
+
+        subdomain = self.options.get('subdomain')
+        if subdomain is None:
+            subdomain = self.blueprint.subdomain
+
+        #: The subdomain that the blueprint should be active for, ``None``
+        #: otherwise.
+        self.subdomain = subdomain
+
+        url_prefix = self.options.get('url_prefix')
+        if url_prefix is None:
+            url_prefix = self.blueprint.url_prefix
+        #: The prefix that should be used for all URLs defined on the
+        #: blueprint.
+        self.url_prefix = url_prefix
+
+        #: A dictionary with URL defaults that is added to each and every
+        #: URL that was defined with the blueprint.
+        self.url_defaults = dict(self.blueprint.url_values_defaults)
+        self.url_defaults.update(self.options.get('url_defaults', ()))
+
+    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
+        """A helper method to register a rule (and optionally a view function)
+        to the application.  The endpoint is automatically prefixed with the
+        blueprint's name.
+        """
+        if self.url_prefix is not None:
+            if rule:
+                rule = '/'.join((
+                    self.url_prefix.rstrip('/'), rule.lstrip('/')))
+            else:
+                rule = self.url_prefix
+        options.setdefault('subdomain', self.subdomain)
+        if endpoint is None:
+            endpoint = _endpoint_from_view_func(view_func)
+        defaults = self.url_defaults
+        if 'defaults' in options:
+            defaults = dict(defaults, **options.pop('defaults'))
+        self.app.add_url_rule(rule, '%s.%s' % (self.blueprint.name, endpoint),
+                              view_func, defaults=defaults, **options)
+
+
+class Blueprint(_PackageBoundObject):
+    """Represents a blueprint.  A blueprint is an object that records
+    functions that will be called with the
+    :class:`~flask.blueprints.BlueprintSetupState` later to register functions
+    or other things on the main application.  See :ref:`blueprints` for more
+    information.
+
+    .. versionadded:: 0.7
+    """
+
+    warn_on_modifications = False
+    _got_registered_once = False
+
+    #: Blueprint local JSON decoder class to use.
+    #: Set to ``None`` to use the app's :class:`~flask.app.Flask.json_encoder`.
+    json_encoder = None
+    #: Blueprint local JSON decoder class to use.
+    #: Set to ``None`` to use the app's :class:`~flask.app.Flask.json_decoder`.
+    json_decoder = None
+
+    # TODO remove the next three attrs when Sphinx :inherited-members: works
+    # https://github.com/sphinx-doc/sphinx/issues/741
+
+    #: The name of the package or module that this app belongs to. Do not
+    #: change this once it is set by the constructor.
+    import_name = None
+
+    #: Location of the template files to be added to the template lookup.
+    #: ``None`` if templates should not be added.
+    template_folder = None
+
+    #: Absolute path to the package on the filesystem. Used to look up
+    #: resources contained in the package.
+    root_path = None
+
+    def __init__(self, name, import_name, static_folder=None,
+                 static_url_path=None, template_folder=None,
+                 url_prefix=None, subdomain=None, url_defaults=None,
+                 root_path=None):
+        _PackageBoundObject.__init__(self, import_name, template_folder,
+                                     root_path=root_path)
+        self.name = name
+        self.url_prefix = url_prefix
+        self.subdomain = subdomain
+        self.static_folder = static_folder
+        self.static_url_path = static_url_path
+        self.deferred_functions = []
+        if url_defaults is None:
+            url_defaults = {}
+        self.url_values_defaults = url_defaults
+
+    def record(self, func):
+        """Registers a function that is called when the blueprint is
+        registered on the application.  This function is called with the
+        state as argument as returned by the :meth:`make_setup_state`
+        method.
+        """
+        if self._got_registered_once and self.warn_on_modifications:
+            from warnings import warn
+            warn(Warning('The blueprint was already registered once '
+                         'but is getting modified now.  These changes '
+                         'will not show up.'))
+        self.deferred_functions.append(func)
+
+    def record_once(self, func):
+        """Works like :meth:`record` but wraps the function in another
+        function that will ensure the function is only called once.  If the
+        blueprint is registered a second time on the application, the
+        function passed is not called.
+        """
+        def wrapper(state):
+            if state.first_registration:
+                func(state)
+        return self.record(update_wrapper(wrapper, func))
+
+    def make_setup_state(self, app, options, first_registration=False):
+        """Creates an instance of :meth:`~flask.blueprints.BlueprintSetupState`
+        object that is later passed to the register callback functions.
+        Subclasses can override this to return a subclass of the setup state.
+        """
+        return BlueprintSetupState(self, app, options, first_registration)
+
+    def register(self, app, options, first_registration=False):
+        """Called by :meth:`Flask.register_blueprint` to register all views
+        and callbacks registered on the blueprint with the application. Creates
+        a :class:`.BlueprintSetupState` and calls each :meth:`record` callback
+        with it.
+
+        :param app: The application this blueprint is being registered with.
+        :param options: Keyword arguments forwarded from
+            :meth:`~Flask.register_blueprint`.
+        :param first_registration: Whether this is the first time this
+            blueprint has been registered on the application.
+        """
+        self._got_registered_once = True
+        state = self.make_setup_state(app, options, first_registration)
+
+        if self.has_static_folder:
+            state.add_url_rule(
+                self.static_url_path + '/<path:filename>',
+                view_func=self.send_static_file, endpoint='static'
+            )
+
+        for deferred in self.deferred_functions:
+            deferred(state)
+
+    def route(self, rule, **options):
+        """Like :meth:`Flask.route` but for a blueprint.  The endpoint for the
+        :func:`url_for` function is prefixed with the name of the blueprint.
+        """
+        def decorator(f):
+            endpoint = options.pop("endpoint", f.__name__)
+            self.add_url_rule(rule, endpoint, f, **options)
+            return f
+        return decorator
+
+    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
+        """Like :meth:`Flask.add_url_rule` but for a blueprint.  The endpoint for
+        the :func:`url_for` function is prefixed with the name of the blueprint.
+        """
+        if endpoint:
+            assert '.' not in endpoint, "Blueprint endpoints should not contain dots"
+        if view_func and hasattr(view_func, '__name__'):
+            assert '.' not in view_func.__name__, "Blueprint view function name should not contain dots"
+        self.record(lambda s:
+            s.add_url_rule(rule, endpoint, view_func, **options))
+
+    def endpoint(self, endpoint):
+        """Like :meth:`Flask.endpoint` but for a blueprint.  This does not
+        prefix the endpoint with the blueprint name, this has to be done
+        explicitly by the user of this method.  If the endpoint is prefixed
+        with a `.` it will be registered to the current blueprint, otherwise
+        it's an application independent endpoint.
+        """
+        def decorator(f):
+            def register_endpoint(state):
+                state.app.view_functions[endpoint] = f
+            self.record_once(register_endpoint)
+            return f
+        return decorator
+
+    def app_template_filter(self, name=None):
+        """Register a custom template filter, available application wide.  Like
+        :meth:`Flask.template_filter` but for a blueprint.
+
+        :param name: the optional name of the filter, otherwise the
+                     function name will be used.
+        """
+        def decorator(f):
+            self.add_app_template_filter(f, name=name)
+            return f
+        return decorator
+
+    def add_app_template_filter(self, f, name=None):
+        """Register a custom template filter, available application wide.  Like
+        :meth:`Flask.add_template_filter` but for a blueprint.  Works exactly
+        like the :meth:`app_template_filter` decorator.
+
+        :param name: the optional name of the filter, otherwise the
+                     function name will be used.
+        """
+        def register_template(state):
+            state.app.jinja_env.filters[name or f.__name__] = f
+        self.record_once(register_template)
+
+    def app_template_test(self, name=None):
+        """Register a custom template test, available application wide.  Like
+        :meth:`Flask.template_test` but for a blueprint.
+
+        .. versionadded:: 0.10
+
+        :param name: the optional name of the test, otherwise the
+                     function name will be used.
+        """
+        def decorator(f):
+            self.add_app_template_test(f, name=name)
+            return f
+        return decorator
+
+    def add_app_template_test(self, f, name=None):
+        """Register a custom template test, available application wide.  Like
+        :meth:`Flask.add_template_test` but for a blueprint.  Works exactly
+        like the :meth:`app_template_test` decorator.
+
+        .. versionadded:: 0.10
+
+        :param name: the optional name of the test, otherwise the
+                     function name will be used.
+        """
+        def register_template(state):
+            state.app.jinja_env.tests[name or f.__name__] = f
+        self.record_once(register_template)
+
+    def app_template_global(self, name=None):
+        """Register a custom template global, available application wide.  Like
+        :meth:`Flask.template_global` but for a blueprint.
+
+        .. versionadded:: 0.10
+
+        :param name: the optional name of the global, otherwise the
+                     function name will be used.
+        """
+        def decorator(f):
+            self.add_app_template_global(f, name=name)
+            return f
+        return decorator
+
+    def add_app_template_global(self, f, name=None):
+        """Register a custom template global, available application wide.  Like
+        :meth:`Flask.add_template_global` but for a blueprint.  Works exactly
+        like the :meth:`app_template_global` decorator.
+
+        .. versionadded:: 0.10
+
+        :param name: the optional name of the global, otherwise the
+                     function name will be used.
+        """
+        def register_template(state):
+            state.app.jinja_env.globals[name or f.__name__] = f
+        self.record_once(register_template)
+
+    def before_request(self, f):
+        """Like :meth:`Flask.before_request` but for a blueprint.  This function
+        is only executed before each request that is handled by a function of
+        that blueprint.
+        """
+        self.record_once(lambda s: s.app.before_request_funcs
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def before_app_request(self, f):
+        """Like :meth:`Flask.before_request`.  Such a function is executed
+        before each request, even if outside of a blueprint.
+        """
+        self.record_once(lambda s: s.app.before_request_funcs
+            .setdefault(None, []).append(f))
+        return f
+
+    def before_app_first_request(self, f):
+        """Like :meth:`Flask.before_first_request`.  Such a function is
+        executed before the first request to the application.
+        """
+        self.record_once(lambda s: s.app.before_first_request_funcs.append(f))
+        return f
+
+    def after_request(self, f):
+        """Like :meth:`Flask.after_request` but for a blueprint.  This function
+        is only executed after each request that is handled by a function of
+        that blueprint.
+        """
+        self.record_once(lambda s: s.app.after_request_funcs
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def after_app_request(self, f):
+        """Like :meth:`Flask.after_request` but for a blueprint.  Such a function
+        is executed after each request, even if outside of the blueprint.
+        """
+        self.record_once(lambda s: s.app.after_request_funcs
+            .setdefault(None, []).append(f))
+        return f
+
+    def teardown_request(self, f):
+        """Like :meth:`Flask.teardown_request` but for a blueprint.  This
+        function is only executed when tearing down requests handled by a
+        function of that blueprint.  Teardown request functions are executed
+        when the request context is popped, even when no actual request was
+        performed.
+        """
+        self.record_once(lambda s: s.app.teardown_request_funcs
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def teardown_app_request(self, f):
+        """Like :meth:`Flask.teardown_request` but for a blueprint.  Such a
+        function is executed when tearing down each request, even if outside of
+        the blueprint.
+        """
+        self.record_once(lambda s: s.app.teardown_request_funcs
+            .setdefault(None, []).append(f))
+        return f
+
+    def context_processor(self, f):
+        """Like :meth:`Flask.context_processor` but for a blueprint.  This
+        function is only executed for requests handled by a blueprint.
+        """
+        self.record_once(lambda s: s.app.template_context_processors
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def app_context_processor(self, f):
+        """Like :meth:`Flask.context_processor` but for a blueprint.  Such a
+        function is executed each request, even if outside of the blueprint.
+        """
+        self.record_once(lambda s: s.app.template_context_processors
+            .setdefault(None, []).append(f))
+        return f
+
+    def app_errorhandler(self, code):
+        """Like :meth:`Flask.errorhandler` but for a blueprint.  This
+        handler is used for all requests, even if outside of the blueprint.
+        """
+        def decorator(f):
+            self.record_once(lambda s: s.app.errorhandler(code)(f))
+            return f
+        return decorator
+
+    def url_value_preprocessor(self, f):
+        """Registers a function as URL value preprocessor for this
+        blueprint.  It's called before the view functions are called and
+        can modify the url values provided.
+        """
+        self.record_once(lambda s: s.app.url_value_preprocessors
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def url_defaults(self, f):
+        """Callback function for URL defaults for this blueprint.  It's called
+        with the endpoint and values and should update the values passed
+        in place.
+        """
+        self.record_once(lambda s: s.app.url_default_functions
+            .setdefault(self.name, []).append(f))
+        return f
+
+    def app_url_value_preprocessor(self, f):
+        """Same as :meth:`url_value_preprocessor` but application wide.
+        """
+        self.record_once(lambda s: s.app.url_value_preprocessors
+            .setdefault(None, []).append(f))
+        return f
+
+    def app_url_defaults(self, f):
+        """Same as :meth:`url_defaults` but application wide.
+        """
+        self.record_once(lambda s: s.app.url_default_functions
+            .setdefault(None, []).append(f))
+        return f
+
+    def errorhandler(self, code_or_exception):
+        """Registers an error handler that becomes active for this blueprint
+        only.  Please be aware that routing does not happen local to a
+        blueprint so an error handler for 404 usually is not handled by
+        a blueprint unless it is caused inside a view function.  Another
+        special case is the 500 internal server error which is always looked
+        up from the application.
+
+        Otherwise works as the :meth:`~flask.Flask.errorhandler` decorator
+        of the :class:`~flask.Flask` object.
+        """
+        def decorator(f):
+            self.record_once(lambda s: s.app._register_error_handler(
+                self.name, code_or_exception, f))
+            return f
+        return decorator
+
+    def register_error_handler(self, code_or_exception, f):
+        """Non-decorator version of the :meth:`errorhandler` error attach
+        function, akin to the :meth:`~flask.Flask.register_error_handler`
+        application-wide function of the :class:`~flask.Flask` object but
+        for error handlers limited to this blueprint.
+
+        .. versionadded:: 0.11
+        """
+        self.record_once(lambda s: s.app._register_error_handler(
+            self.name, code_or_exception, f))

python/flask/cli.py

@@ -0,0 +1,910 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.cli
+    ~~~~~~~~~
+
+    A simple command line application to run flask apps.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import print_function
+
+import ast
+import inspect
+import os
+import platform
+import re
+import ssl
+import sys
+import traceback
+from functools import update_wrapper
+from operator import attrgetter
+from threading import Lock, Thread
+
+import click
+from werkzeug.utils import import_string
+
+from . import __version__
+from ._compat import getargspec, iteritems, reraise, text_type
+from .globals import current_app
+from .helpers import get_debug_flag, get_env, get_load_dotenv
+
+try:
+    import dotenv
+except ImportError:
+    dotenv = None
+
+
+class NoAppException(click.UsageError):
+    """Raised if an application cannot be found or loaded."""
+
+
+def find_best_app(script_info, module):
+    """Given a module instance this tries to find the best possible
+    application in the module or raises an exception.
+    """
+    from . import Flask
+
+    # Search for the most common names first.
+    for attr_name in ('app', 'application'):
+        app = getattr(module, attr_name, None)
+
+        if isinstance(app, Flask):
+            return app
+
+    # Otherwise find the only object that is a Flask instance.
+    matches = [
+        v for k, v in iteritems(module.__dict__) if isinstance(v, Flask)
+    ]
+
+    if len(matches) == 1:
+        return matches[0]
+    elif len(matches) > 1:
+        raise NoAppException(
+            'Detected multiple Flask applications in module "{module}". Use '
+            '"FLASK_APP={module}:name" to specify the correct '
+            'one.'.format(module=module.__name__)
+        )
+
+    # Search for app factory functions.
+    for attr_name in ('create_app', 'make_app'):
+        app_factory = getattr(module, attr_name, None)
+
+        if inspect.isfunction(app_factory):
+            try:
+                app = call_factory(script_info, app_factory)
+
+                if isinstance(app, Flask):
+                    return app
+            except TypeError:
+                if not _called_with_wrong_args(app_factory):
+                    raise
+                raise NoAppException(
+                    'Detected factory "{factory}" in module "{module}", but '
+                    'could not call it without arguments. Use '
+                    '"FLASK_APP=\'{module}:{factory}(args)\'" to specify '
+                    'arguments.'.format(
+                        factory=attr_name, module=module.__name__
+                    )
+                )
+
+    raise NoAppException(
+        'Failed to find Flask application or factory in module "{module}". '
+        'Use "FLASK_APP={module}:name to specify one.'.format(
+            module=module.__name__
+        )
+    )
+
+
+def call_factory(script_info, app_factory, arguments=()):
+    """Takes an app factory, a ``script_info` object and  optionally a tuple
+    of arguments. Checks for the existence of a script_info argument and calls
+    the app_factory depending on that and the arguments provided.
+    """
+    args_spec = getargspec(app_factory)
+    arg_names = args_spec.args
+    arg_defaults = args_spec.defaults
+
+    if 'script_info' in arg_names:
+        return app_factory(*arguments, script_info=script_info)
+    elif arguments:
+        return app_factory(*arguments)
+    elif not arguments and len(arg_names) == 1 and arg_defaults is None:
+        return app_factory(script_info)
+
+    return app_factory()
+
+
+def _called_with_wrong_args(factory):
+    """Check whether calling a function raised a ``TypeError`` because
+    the call failed or because something in the factory raised the
+    error.
+
+    :param factory: the factory function that was called
+    :return: true if the call failed
+    """
+    tb = sys.exc_info()[2]
+
+    try:
+        while tb is not None:
+            if tb.tb_frame.f_code is factory.__code__:
+                # in the factory, it was called successfully
+                return False
+
+            tb = tb.tb_next
+
+        # didn't reach the factory
+        return True
+    finally:
+        del tb
+
+
+def find_app_by_string(script_info, module, app_name):
+    """Checks if the given string is a variable name or a function. If it is a
+    function, it checks for specified arguments and whether it takes a
+    ``script_info`` argument and calls the function with the appropriate
+    arguments.
+    """
+    from flask import Flask
+    match = re.match(r'^ *([^ ()]+) *(?:\((.*?) *,? *\))? *$', app_name)
+
+    if not match:
+        raise NoAppException(
+            '"{name}" is not a valid variable name or function '
+            'expression.'.format(name=app_name)
+        )
+
+    name, args = match.groups()
+
+    try:
+        attr = getattr(module, name)
+    except AttributeError as e:
+        raise NoAppException(e.args[0])
+
+    if inspect.isfunction(attr):
+        if args:
+            try:
+                args = ast.literal_eval('({args},)'.format(args=args))
+            except (ValueError, SyntaxError)as e:
+                raise NoAppException(
+                    'Could not parse the arguments in '
+                    '"{app_name}".'.format(e=e, app_name=app_name)
+                )
+        else:
+            args = ()
+
+        try:
+            app = call_factory(script_info, attr, args)
+        except TypeError as e:
+            if not _called_with_wrong_args(attr):
+                raise
+
+            raise NoAppException(
+                '{e}\nThe factory "{app_name}" in module "{module}" could not '
+                'be called with the specified arguments.'.format(
+                    e=e, app_name=app_name, module=module.__name__
+                )
+            )
+    else:
+        app = attr
+
+    if isinstance(app, Flask):
+        return app
+
+    raise NoAppException(
+        'A valid Flask application was not obtained from '
+        '"{module}:{app_name}".'.format(
+            module=module.__name__, app_name=app_name
+        )
+    )
+
+
+def prepare_import(path):
+    """Given a filename this will try to calculate the python path, add it
+    to the search path and return the actual module name that is expected.
+    """
+    path = os.path.realpath(path)
+
+    if os.path.splitext(path)[1] == '.py':
+        path = os.path.splitext(path)[0]
+
+    if os.path.basename(path) == '__init__':
+        path = os.path.dirname(path)
+
+    module_name = []
+
+    # move up until outside package structure (no __init__.py)
+    while True:
+        path, name = os.path.split(path)
+        module_name.append(name)
+
+        if not os.path.exists(os.path.join(path, '__init__.py')):
+            break
+
+    if sys.path[0] != path:
+        sys.path.insert(0, path)
+
+    return '.'.join(module_name[::-1])
+
+
+def locate_app(script_info, module_name, app_name, raise_if_not_found=True):
+    __traceback_hide__ = True
+
+    try:
+        __import__(module_name)
+    except ImportError:
+        # Reraise the ImportError if it occurred within the imported module.
+        # Determine this by checking whether the trace has a depth > 1.
+        if sys.exc_info()[-1].tb_next:
+            raise NoAppException(
+                'While importing "{name}", an ImportError was raised:'
+                '\n\n{tb}'.format(name=module_name, tb=traceback.format_exc())
+            )
+        elif raise_if_not_found:
+            raise NoAppException(
+                'Could not import "{name}".'.format(name=module_name)
+            )
+        else:
+            return
+
+    module = sys.modules[module_name]
+
+    if app_name is None:
+        return find_best_app(script_info, module)
+    else:
+        return find_app_by_string(script_info, module, app_name)
+
+
+def get_version(ctx, param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    import werkzeug
+    message = (
+        'Python %(python)s\n'
+        'Flask %(flask)s\n'
+        'Werkzeug %(werkzeug)s'
+    )
+    click.echo(message % {
+        'python': platform.python_version(),
+        'flask': __version__,
+        'werkzeug': werkzeug.__version__,
+    }, color=ctx.color)
+    ctx.exit()
+
+
+version_option = click.Option(
+    ['--version'],
+    help='Show the flask version',
+    expose_value=False,
+    callback=get_version,
+    is_flag=True,
+    is_eager=True
+)
+
+
+class DispatchingApp(object):
+    """Special application that dispatches to a Flask application which
+    is imported by name in a background thread.  If an error happens
+    it is recorded and shown as part of the WSGI handling which in case
+    of the Werkzeug debugger means that it shows up in the browser.
+    """
+
+    def __init__(self, loader, use_eager_loading=False):
+        self.loader = loader
+        self._app = None
+        self._lock = Lock()
+        self._bg_loading_exc_info = None
+        if use_eager_loading:
+            self._load_unlocked()
+        else:
+            self._load_in_background()
+
+    def _load_in_background(self):
+        def _load_app():
+            __traceback_hide__ = True
+            with self._lock:
+                try:
+                    self._load_unlocked()
+                except Exception:
+                    self._bg_loading_exc_info = sys.exc_info()
+        t = Thread(target=_load_app, args=())
+        t.start()
+
+    def _flush_bg_loading_exception(self):
+        __traceback_hide__ = True
+        exc_info = self._bg_loading_exc_info
+        if exc_info is not None:
+            self._bg_loading_exc_info = None
+            reraise(*exc_info)
+
+    def _load_unlocked(self):
+        __traceback_hide__ = True
+        self._app = rv = self.loader()
+        self._bg_loading_exc_info = None
+        return rv
+
+    def __call__(self, environ, start_response):
+        __traceback_hide__ = True
+        if self._app is not None:
+            return self._app(environ, start_response)
+        self._flush_bg_loading_exception()
+        with self._lock:
+            if self._app is not None:
+                rv = self._app
+            else:
+                rv = self._load_unlocked()
+            return rv(environ, start_response)
+
+
+class ScriptInfo(object):
+    """Helper object to deal with Flask applications.  This is usually not
+    necessary to interface with as it's used internally in the dispatching
+    to click.  In future versions of Flask this object will most likely play
+    a bigger role.  Typically it's created automatically by the
+    :class:`FlaskGroup` but you can also manually create it and pass it
+    onwards as click object.
+    """
+
+    def __init__(self, app_import_path=None, create_app=None,
+                 set_debug_flag=True):
+        #: Optionally the import path for the Flask application.
+        self.app_import_path = app_import_path or os.environ.get('FLASK_APP')
+        #: Optionally a function that is passed the script info to create
+        #: the instance of the application.
+        self.create_app = create_app
+        #: A dictionary with arbitrary data that can be associated with
+        #: this script info.
+        self.data = {}
+        self.set_debug_flag = set_debug_flag
+        self._loaded_app = None
+
+    def load_app(self):
+        """Loads the Flask app (if not yet loaded) and returns it.  Calling
+        this multiple times will just result in the already loaded app to
+        be returned.
+        """
+        __traceback_hide__ = True
+
+        if self._loaded_app is not None:
+            return self._loaded_app
+
+        app = None
+
+        if self.create_app is not None:
+            app = call_factory(self, self.create_app)
+        else:
+            if self.app_import_path:
+                path, name = (re.split(r':(?![\\/])', self.app_import_path, 1) + [None])[:2]
+                import_name = prepare_import(path)
+                app = locate_app(self, import_name, name)
+            else:
+                for path in ('wsgi.py', 'app.py'):
+                    import_name = prepare_import(path)
+                    app = locate_app(self, import_name, None,
+                                     raise_if_not_found=False)
+
+                    if app:
+                        break
+
+        if not app:
+            raise NoAppException(
+                'Could not locate a Flask application. You did not provide '
+                'the "FLASK_APP" environment variable, and a "wsgi.py" or '
+                '"app.py" module was not found in the current directory.'
+            )
+
+        if self.set_debug_flag:
+            # Update the app's debug flag through the descriptor so that
+            # other values repopulate as well.
+            app.debug = get_debug_flag()
+
+        self._loaded_app = app
+        return app
+
+
+pass_script_info = click.make_pass_decorator(ScriptInfo, ensure=True)
+
+
+def with_appcontext(f):
+    """Wraps a callback so that it's guaranteed to be executed with the
+    script's application context.  If callbacks are registered directly
+    to the ``app.cli`` object then they are wrapped with this function
+    by default unless it's disabled.
+    """
+    @click.pass_context
+    def decorator(__ctx, *args, **kwargs):
+        with __ctx.ensure_object(ScriptInfo).load_app().app_context():
+            return __ctx.invoke(f, *args, **kwargs)
+    return update_wrapper(decorator, f)
+
+
+class AppGroup(click.Group):
+    """This works similar to a regular click :class:`~click.Group` but it
+    changes the behavior of the :meth:`command` decorator so that it
+    automatically wraps the functions in :func:`with_appcontext`.
+
+    Not to be confused with :class:`FlaskGroup`.
+    """
+
+    def command(self, *args, **kwargs):
+        """This works exactly like the method of the same name on a regular
+        :class:`click.Group` but it wraps callbacks in :func:`with_appcontext`
+        unless it's disabled by passing ``with_appcontext=False``.
+        """
+        wrap_for_ctx = kwargs.pop('with_appcontext', True)
+        def decorator(f):
+            if wrap_for_ctx:
+                f = with_appcontext(f)
+            return click.Group.command(self, *args, **kwargs)(f)
+        return decorator
+
+    def group(self, *args, **kwargs):
+        """This works exactly like the method of the same name on a regular
+        :class:`click.Group` but it defaults the group class to
+        :class:`AppGroup`.
+        """
+        kwargs.setdefault('cls', AppGroup)
+        return click.Group.group(self, *args, **kwargs)
+
+
+class FlaskGroup(AppGroup):
+    """Special subclass of the :class:`AppGroup` group that supports
+    loading more commands from the configured Flask app.  Normally a
+    developer does not have to interface with this class but there are
+    some very advanced use cases for which it makes sense to create an
+    instance of this.
+
+    For information as of why this is useful see :ref:`custom-scripts`.
+
+    :param add_default_commands: if this is True then the default run and
+        shell commands wil be added.
+    :param add_version_option: adds the ``--version`` option.
+    :param create_app: an optional callback that is passed the script info and
+        returns the loaded app.
+    :param load_dotenv: Load the nearest :file:`.env` and :file:`.flaskenv`
+        files to set environment variables. Will also change the working
+        directory to the directory containing the first file found.
+    :param set_debug_flag: Set the app's debug flag based on the active
+        environment
+
+    .. versionchanged:: 1.0
+        If installed, python-dotenv will be used to load environment variables
+        from :file:`.env` and :file:`.flaskenv` files.
+    """
+
+    def __init__(self, add_default_commands=True, create_app=None,
+                 add_version_option=True, load_dotenv=True,
+                 set_debug_flag=True, **extra):
+        params = list(extra.pop('params', None) or ())
+
+        if add_version_option:
+            params.append(version_option)
+
+        AppGroup.__init__(self, params=params, **extra)
+        self.create_app = create_app
+        self.load_dotenv = load_dotenv
+        self.set_debug_flag = set_debug_flag
+
+        if add_default_commands:
+            self.add_command(run_command)
+            self.add_command(shell_command)
+            self.add_command(routes_command)
+
+        self._loaded_plugin_commands = False
+
+    def _load_plugin_commands(self):
+        if self._loaded_plugin_commands:
+            return
+        try:
+            import pkg_resources
+        except ImportError:
+            self._loaded_plugin_commands = True
+            return
+
+        for ep in pkg_resources.iter_entry_points('flask.commands'):
+            self.add_command(ep.load(), ep.name)
+        self._loaded_plugin_commands = True
+
+    def get_command(self, ctx, name):
+        self._load_plugin_commands()
+
+        # We load built-in commands first as these should always be the
+        # same no matter what the app does.  If the app does want to
+        # override this it needs to make a custom instance of this group
+        # and not attach the default commands.
+        #
+        # This also means that the script stays functional in case the
+        # application completely fails.
+        rv = AppGroup.get_command(self, ctx, name)
+        if rv is not None:
+            return rv
+
+        info = ctx.ensure_object(ScriptInfo)
+        try:
+            rv = info.load_app().cli.get_command(ctx, name)
+            if rv is not None:
+                return rv
+        except NoAppException:
+            pass
+
+    def list_commands(self, ctx):
+        self._load_plugin_commands()
+
+        # The commands available is the list of both the application (if
+        # available) plus the builtin commands.
+        rv = set(click.Group.list_commands(self, ctx))
+        info = ctx.ensure_object(ScriptInfo)
+        try:
+            rv.update(info.load_app().cli.list_commands(ctx))
+        except Exception:
+            # Here we intentionally swallow all exceptions as we don't
+            # want the help page to break if the app does not exist.
+            # If someone attempts to use the command we try to create
+            # the app again and this will give us the error.
+            # However, we will not do so silently because that would confuse
+            # users.
+            traceback.print_exc()
+        return sorted(rv)
+
+    def main(self, *args, **kwargs):
+        # Set a global flag that indicates that we were invoked from the
+        # command line interface. This is detected by Flask.run to make the
+        # call into a no-op. This is necessary to avoid ugly errors when the
+        # script that is loaded here also attempts to start a server.
+        os.environ['FLASK_RUN_FROM_CLI'] = 'true'
+
+        if get_load_dotenv(self.load_dotenv):
+            load_dotenv()
+
+        obj = kwargs.get('obj')
+
+        if obj is None:
+            obj = ScriptInfo(create_app=self.create_app,
+                             set_debug_flag=self.set_debug_flag)
+
+        kwargs['obj'] = obj
+        kwargs.setdefault('auto_envvar_prefix', 'FLASK')
+        return super(FlaskGroup, self).main(*args, **kwargs)
+
+
+def _path_is_ancestor(path, other):
+    """Take ``other`` and remove the length of ``path`` from it. Then join it
+    to ``path``. If it is the original value, ``path`` is an ancestor of
+    ``other``."""
+    return os.path.join(path, other[len(path):].lstrip(os.sep)) == other
+
+
+def load_dotenv(path=None):
+    """Load "dotenv" files in order of precedence to set environment variables.
+
+    If an env var is already set it is not overwritten, so earlier files in the
+    list are preferred over later files.
+
+    Changes the current working directory to the location of the first file
+    found, with the assumption that it is in the top level project directory
+    and will be where the Python path should import local packages from.
+
+    This is a no-op if `python-dotenv`_ is not installed.
+
+    .. _python-dotenv: https://github.com/theskumar/python-dotenv#readme
+
+    :param path: Load the file at this location instead of searching.
+    :return: ``True`` if a file was loaded.
+
+    .. versionadded:: 1.0
+    """
+    if dotenv is None:
+        if path or os.path.exists('.env') or os.path.exists('.flaskenv'):
+            click.secho(
+                ' * Tip: There are .env files present.'
+                ' Do "pip install python-dotenv" to use them.',
+                fg='yellow')
+        return
+
+    if path is not None:
+        return dotenv.load_dotenv(path)
+
+    new_dir = None
+
+    for name in ('.env', '.flaskenv'):
+        path = dotenv.find_dotenv(name, usecwd=True)
+
+        if not path:
+            continue
+
+        if new_dir is None:
+            new_dir = os.path.dirname(path)
+
+        dotenv.load_dotenv(path)
+
+    if new_dir and os.getcwd() != new_dir:
+        os.chdir(new_dir)
+
+    return new_dir is not None  # at least one file was located and loaded
+
+
+def show_server_banner(env, debug, app_import_path, eager_loading):
+    """Show extra startup messages the first time the server is run,
+    ignoring the reloader.
+    """
+    if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
+        return
+
+    if app_import_path is not None:
+        message = ' * Serving Flask app "{0}"'.format(app_import_path)
+
+        if not eager_loading:
+            message += ' (lazy loading)'
+
+        click.echo(message)
+
+    click.echo(' * Environment: {0}'.format(env))
+
+    if env == 'production':
+        click.secho(
+            '   WARNING: This is a development server. '
+            'Do not use it in a production deployment.', fg='red')
+        click.secho('   Use a production WSGI server instead.', dim=True)
+
+    if debug is not None:
+        click.echo(' * Debug mode: {0}'.format('on' if debug else 'off'))
+
+
+class CertParamType(click.ParamType):
+    """Click option type for the ``--cert`` option. Allows either an
+    existing file, the string ``'adhoc'``, or an import for a
+    :class:`~ssl.SSLContext` object.
+    """
+
+    name = 'path'
+
+    def __init__(self):
+        self.path_type = click.Path(
+            exists=True, dir_okay=False, resolve_path=True)
+
+    def convert(self, value, param, ctx):
+        try:
+            return self.path_type(value, param, ctx)
+        except click.BadParameter:
+            value = click.STRING(value, param, ctx).lower()
+
+            if value == 'adhoc':
+                try:
+                    import OpenSSL
+                except ImportError:
+                    raise click.BadParameter(
+                        'Using ad-hoc certificates requires pyOpenSSL.',
+                        ctx, param)
+
+                return value
+
+            obj = import_string(value, silent=True)
+
+            if sys.version_info < (2, 7, 9):
+                if obj:
+                    return obj
+            else:
+                if isinstance(obj, ssl.SSLContext):
+                    return obj
+
+            raise
+
+
+def _validate_key(ctx, param, value):
+    """The ``--key`` option must be specified when ``--cert`` is a file.
+    Modifies the ``cert`` param to be a ``(cert, key)`` pair if needed.
+    """
+    cert = ctx.params.get('cert')
+    is_adhoc = cert == 'adhoc'
+
+    if sys.version_info < (2, 7, 9):
+        is_context = cert and not isinstance(cert, (text_type, bytes))
+    else:
+        is_context = isinstance(cert, ssl.SSLContext)
+
+    if value is not None:
+        if is_adhoc:
+            raise click.BadParameter(
+                'When "--cert" is "adhoc", "--key" is not used.',
+                ctx, param)
+
+        if is_context:
+            raise click.BadParameter(
+                'When "--cert" is an SSLContext object, "--key is not used.',
+                ctx, param)
+
+        if not cert:
+            raise click.BadParameter(
+                '"--cert" must also be specified.',
+                ctx, param)
+
+        ctx.params['cert'] = cert, value
+
+    else:
+        if cert and not (is_adhoc or is_context):
+            raise click.BadParameter(
+                'Required when using "--cert".',
+                ctx, param)
+
+    return value
+
+
+@click.command('run', short_help='Run a development server.')
+@click.option('--host', '-h', default='127.0.0.1',
+              help='The interface to bind to.')
+@click.option('--port', '-p', default=5000,
+              help='The port to bind to.')
+@click.option('--cert', type=CertParamType(),
+              help='Specify a certificate file to use HTTPS.')
+@click.option('--key',
+              type=click.Path(exists=True, dir_okay=False, resolve_path=True),
+              callback=_validate_key, expose_value=False,
+              help='The key file to use when specifying a certificate.')
+@click.option('--reload/--no-reload', default=None,
+              help='Enable or disable the reloader. By default the reloader '
+              'is active if debug is enabled.')
+@click.option('--debugger/--no-debugger', default=None,
+              help='Enable or disable the debugger. By default the debugger '
+              'is active if debug is enabled.')
+@click.option('--eager-loading/--lazy-loader', default=None,
+              help='Enable or disable eager loading. By default eager '
+              'loading is enabled if the reloader is disabled.')
+@click.option('--with-threads/--without-threads', default=True,
+              help='Enable or disable multithreading.')
+@pass_script_info
+def run_command(info, host, port, reload, debugger, eager_loading,
+                with_threads, cert):
+    """Run a local development server.
+
+    This server is for development purposes only. It does not provide
+    the stability, security, or performance of production WSGI servers.
+
+    The reloader and debugger are enabled by default if
+    FLASK_ENV=development or FLASK_DEBUG=1.
+    """
+    debug = get_debug_flag()
+
+    if reload is None:
+        reload = debug
+
+    if debugger is None:
+        debugger = debug
+
+    if eager_loading is None:
+        eager_loading = not reload
+
+    show_server_banner(get_env(), debug, info.app_import_path, eager_loading)
+    app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
+
+    from werkzeug.serving import run_simple
+    run_simple(host, port, app, use_reloader=reload, use_debugger=debugger,
+               threaded=with_threads, ssl_context=cert)
+
+
+@click.command('shell', short_help='Run a shell in the app context.')
+@with_appcontext
+def shell_command():
+    """Run an interactive Python shell in the context of a given
+    Flask application.  The application will populate the default
+    namespace of this shell according to it's configuration.
+
+    This is useful for executing small snippets of management code
+    without having to manually configure the application.
+    """
+    import code
+    from flask.globals import _app_ctx_stack
+    app = _app_ctx_stack.top.app
+    banner = 'Python %s on %s\nApp: %s [%s]\nInstance: %s' % (
+        sys.version,
+        sys.platform,
+        app.import_name,
+        app.env,
+        app.instance_path,
+    )
+    ctx = {}
+
+    # Support the regular Python interpreter startup script if someone
+    # is using it.
+    startup = os.environ.get('PYTHONSTARTUP')
+    if startup and os.path.isfile(startup):
+        with open(startup, 'r') as f:
+            eval(compile(f.read(), startup, 'exec'), ctx)
+
+    ctx.update(app.make_shell_context())
+
+    code.interact(banner=banner, local=ctx)
+
+
+@click.command('routes', short_help='Show the routes for the app.')
+@click.option(
+    '--sort', '-s',
+    type=click.Choice(('endpoint', 'methods', 'rule', 'match')),
+    default='endpoint',
+    help=(
+        'Method to sort routes by. "match" is the order that Flask will match '
+        'routes when dispatching a request.'
+    )
+)
+@click.option(
+    '--all-methods',
+    is_flag=True,
+    help="Show HEAD and OPTIONS methods."
+)
+@with_appcontext
+def routes_command(sort, all_methods):
+    """Show all registered routes with endpoints and methods."""
+
+    rules = list(current_app.url_map.iter_rules())
+    if not rules:
+        click.echo('No routes were registered.')
+        return
+
+    ignored_methods = set(() if all_methods else ('HEAD', 'OPTIONS'))
+
+    if sort in ('endpoint', 'rule'):
+        rules = sorted(rules, key=attrgetter(sort))
+    elif sort == 'methods':
+        rules = sorted(rules, key=lambda rule: sorted(rule.methods))
+
+    rule_methods = [
+        ', '.join(sorted(rule.methods - ignored_methods)) for rule in rules
+    ]
+
+    headers = ('Endpoint', 'Methods', 'Rule')
+    widths = (
+        max(len(rule.endpoint) for rule in rules),
+        max(len(methods) for methods in rule_methods),
+        max(len(rule.rule) for rule in rules),
+    )
+    widths = [max(len(h), w) for h, w in zip(headers, widths)]
+    row = '{{0:<{0}}}  {{1:<{1}}}  {{2:<{2}}}'.format(*widths)
+
+    click.echo(row.format(*headers).strip())
+    click.echo(row.format(*('-' * width for width in widths)))
+
+    for rule, methods in zip(rules, rule_methods):
+        click.echo(row.format(rule.endpoint, methods, rule.rule).rstrip())
+
+
+cli = FlaskGroup(help="""\
+A general utility script for Flask applications.
+
+Provides commands from Flask, extensions, and the application. Loads the
+application defined in the FLASK_APP environment variable, or from a wsgi.py
+file. Setting the FLASK_ENV environment variable to 'development' will enable
+debug mode.
+
+\b
+  {prefix}{cmd} FLASK_APP=hello.py
+  {prefix}{cmd} FLASK_ENV=development
+  {prefix}flask run
+""".format(
+    cmd='export' if os.name == 'posix' else 'set',
+    prefix='$ ' if os.name == 'posix' else '> '
+))
+
+
+def main(as_module=False):
+    args = sys.argv[1:]
+
+    if as_module:
+        this_module = 'flask'
+
+        if sys.version_info < (2, 7):
+            this_module += '.cli'
+
+        name = 'python -m ' + this_module
+
+        # Python rewrites "python -m flask" to the path to the file in argv.
+        # Restore the original command so that the reloader works.
+        sys.argv = ['-m', this_module] + args
+    else:
+        name = None
+
+    cli.main(args=args, prog_name=name)
+
+
+if __name__ == '__main__':
+    main(as_module=True)

python/flask/config.py

@@ -0,0 +1,269 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.config
+    ~~~~~~~~~~~~
+
+    Implements the configuration related objects.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+import types
+import errno
+
+from werkzeug.utils import import_string
+from ._compat import string_types, iteritems
+from . import json
+
+
+class ConfigAttribute(object):
+    """Makes an attribute forward to the config"""
+
+    def __init__(self, name, get_converter=None):
+        self.__name__ = name
+        self.get_converter = get_converter
+
+    def __get__(self, obj, type=None):
+        if obj is None:
+            return self
+        rv = obj.config[self.__name__]
+        if self.get_converter is not None:
+            rv = self.get_converter(rv)
+        return rv
+
+    def __set__(self, obj, value):
+        obj.config[self.__name__] = value
+
+
+class Config(dict):
+    """Works exactly like a dict but provides ways to fill it from files
+    or special dictionaries.  There are two common patterns to populate the
+    config.
+
+    Either you can fill the config from a config file::
+
+        app.config.from_pyfile('yourconfig.cfg')
+
+    Or alternatively you can define the configuration options in the
+    module that calls :meth:`from_object` or provide an import path to
+    a module that should be loaded.  It is also possible to tell it to
+    use the same module and with that provide the configuration values
+    just before the call::
+
+        DEBUG = True
+        SECRET_KEY = 'development key'
+        app.config.from_object(__name__)
+
+    In both cases (loading from any Python file or loading from modules),
+    only uppercase keys are added to the config.  This makes it possible to use
+    lowercase values in the config file for temporary values that are not added
+    to the config or to define the config keys in the same file that implements
+    the application.
+
+    Probably the most interesting way to load configurations is from an
+    environment variable pointing to a file::
+
+        app.config.from_envvar('YOURAPPLICATION_SETTINGS')
+
+    In this case before launching the application you have to set this
+    environment variable to the file you want to use.  On Linux and OS X
+    use the export statement::
+
+        export YOURAPPLICATION_SETTINGS='/path/to/config/file'
+
+    On windows use `set` instead.
+
+    :param root_path: path to which files are read relative from.  When the
+                      config object is created by the application, this is
+                      the application's :attr:`~flask.Flask.root_path`.
+    :param defaults: an optional dictionary of default values
+    """
+
+    def __init__(self, root_path, defaults=None):
+        dict.__init__(self, defaults or {})
+        self.root_path = root_path
+
+    def from_envvar(self, variable_name, silent=False):
+        """Loads a configuration from an environment variable pointing to
+        a configuration file.  This is basically just a shortcut with nicer
+        error messages for this line of code::
+
+            app.config.from_pyfile(os.environ['YOURAPPLICATION_SETTINGS'])
+
+        :param variable_name: name of the environment variable
+        :param silent: set to ``True`` if you want silent failure for missing
+                       files.
+        :return: bool. ``True`` if able to load config, ``False`` otherwise.
+        """
+        rv = os.environ.get(variable_name)
+        if not rv:
+            if silent:
+                return False
+            raise RuntimeError('The environment variable %r is not set '
+                               'and as such configuration could not be '
+                               'loaded.  Set this variable and make it '
+                               'point to a configuration file' %
+                               variable_name)
+        return self.from_pyfile(rv, silent=silent)
+
+    def from_pyfile(self, filename, silent=False):
+        """Updates the values in the config from a Python file.  This function
+        behaves as if the file was imported as module with the
+        :meth:`from_object` function.
+
+        :param filename: the filename of the config.  This can either be an
+                         absolute filename or a filename relative to the
+                         root path.
+        :param silent: set to ``True`` if you want silent failure for missing
+                       files.
+
+        .. versionadded:: 0.7
+           `silent` parameter.
+        """
+        filename = os.path.join(self.root_path, filename)
+        d = types.ModuleType('config')
+        d.__file__ = filename
+        try:
+            with open(filename, mode='rb') as config_file:
+                exec(compile(config_file.read(), filename, 'exec'), d.__dict__)
+        except IOError as e:
+            if silent and e.errno in (
+                errno.ENOENT, errno.EISDIR, errno.ENOTDIR
+            ):
+                return False
+            e.strerror = 'Unable to load configuration file (%s)' % e.strerror
+            raise
+        self.from_object(d)
+        return True
+
+    def from_object(self, obj):
+        """Updates the values from the given object.  An object can be of one
+        of the following two types:
+
+        -   a string: in this case the object with that name will be imported
+        -   an actual object reference: that object is used directly
+
+        Objects are usually either modules or classes. :meth:`from_object`
+        loads only the uppercase attributes of the module/class. A ``dict``
+        object will not work with :meth:`from_object` because the keys of a
+        ``dict`` are not attributes of the ``dict`` class.
+
+        Example of module-based configuration::
+
+            app.config.from_object('yourapplication.default_config')
+            from yourapplication import default_config
+            app.config.from_object(default_config)
+
+        Nothing is done to the object before loading. If the object is a
+        class and has ``@property`` attributes, it needs to be
+        instantiated before being passed to this method.
+
+        You should not use this function to load the actual configuration but
+        rather configuration defaults.  The actual config should be loaded
+        with :meth:`from_pyfile` and ideally from a location not within the
+        package because the package might be installed system wide.
+
+        See :ref:`config-dev-prod` for an example of class-based configuration
+        using :meth:`from_object`.
+
+        :param obj: an import name or object
+        """
+        if isinstance(obj, string_types):
+            obj = import_string(obj)
+        for key in dir(obj):
+            if key.isupper():
+                self[key] = getattr(obj, key)
+
+    def from_json(self, filename, silent=False):
+        """Updates the values in the config from a JSON file. This function
+        behaves as if the JSON object was a dictionary and passed to the
+        :meth:`from_mapping` function.
+
+        :param filename: the filename of the JSON file.  This can either be an
+                         absolute filename or a filename relative to the
+                         root path.
+        :param silent: set to ``True`` if you want silent failure for missing
+                       files.
+
+        .. versionadded:: 0.11
+        """
+        filename = os.path.join(self.root_path, filename)
+
+        try:
+            with open(filename) as json_file:
+                obj = json.loads(json_file.read())
+        except IOError as e:
+            if silent and e.errno in (errno.ENOENT, errno.EISDIR):
+                return False
+            e.strerror = 'Unable to load configuration file (%s)' % e.strerror
+            raise
+        return self.from_mapping(obj)
+
+    def from_mapping(self, *mapping, **kwargs):
+        """Updates the config like :meth:`update` ignoring items with non-upper
+        keys.
+
+        .. versionadded:: 0.11
+        """
+        mappings = []
+        if len(mapping) == 1:
+            if hasattr(mapping[0], 'items'):
+                mappings.append(mapping[0].items())
+            else:
+                mappings.append(mapping[0])
+        elif len(mapping) > 1:
+            raise TypeError(
+                'expected at most 1 positional argument, got %d' % len(mapping)
+            )
+        mappings.append(kwargs.items())
+        for mapping in mappings:
+            for (key, value) in mapping:
+                if key.isupper():
+                    self[key] = value
+        return True
+
+    def get_namespace(self, namespace, lowercase=True, trim_namespace=True):
+        """Returns a dictionary containing a subset of configuration options
+        that match the specified namespace/prefix. Example usage::
+
+            app.config['IMAGE_STORE_TYPE'] = 'fs'
+            app.config['IMAGE_STORE_PATH'] = '/var/app/images'
+            app.config['IMAGE_STORE_BASE_URL'] = 'http://img.website.com'
+            image_store_config = app.config.get_namespace('IMAGE_STORE_')
+
+        The resulting dictionary `image_store_config` would look like::
+
+            {
+                'type': 'fs',
+                'path': '/var/app/images',
+                'base_url': 'http://img.website.com'
+            }
+
+        This is often useful when configuration options map directly to
+        keyword arguments in functions or class constructors.
+
+        :param namespace: a configuration namespace
+        :param lowercase: a flag indicating if the keys of the resulting
+                          dictionary should be lowercase
+        :param trim_namespace: a flag indicating if the keys of the resulting
+                          dictionary should not include the namespace
+
+        .. versionadded:: 0.11
+        """
+        rv = {}
+        for k, v in iteritems(self):
+            if not k.startswith(namespace):
+                continue
+            if trim_namespace:
+                key = k[len(namespace):]
+            else:
+                key = k
+            if lowercase:
+                key = key.lower()
+            rv[key] = v
+        return rv
+
+    def __repr__(self):
+        return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self))

python/flask/ctx.py

@@ -0,0 +1,457 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.ctx
+    ~~~~~~~~~
+
+    Implements the objects required to keep the context.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import sys
+from functools import update_wrapper
+
+from werkzeug.exceptions import HTTPException
+
+from .globals import _request_ctx_stack, _app_ctx_stack
+from .signals import appcontext_pushed, appcontext_popped
+from ._compat import BROKEN_PYPY_CTXMGR_EXIT, reraise
+
+
+# a singleton sentinel value for parameter defaults
+_sentinel = object()
+
+
+class _AppCtxGlobals(object):
+    """A plain object. Used as a namespace for storing data during an
+    application context.
+
+    Creating an app context automatically creates this object, which is
+    made available as the :data:`g` proxy.
+
+    .. describe:: 'key' in g
+
+        Check whether an attribute is present.
+
+        .. versionadded:: 0.10
+
+    .. describe:: iter(g)
+
+        Return an iterator over the attribute names.
+
+        .. versionadded:: 0.10
+    """
+
+    def get(self, name, default=None):
+        """Get an attribute by name, or a default value. Like
+        :meth:`dict.get`.
+
+        :param name: Name of attribute to get.
+        :param default: Value to return if the attribute is not present.
+
+        .. versionadded:: 0.10
+        """
+        return self.__dict__.get(name, default)
+
+    def pop(self, name, default=_sentinel):
+        """Get and remove an attribute by name. Like :meth:`dict.pop`.
+
+        :param name: Name of attribute to pop.
+        :param default: Value to return if the attribute is not present,
+            instead of raise a ``KeyError``.
+
+        .. versionadded:: 0.11
+        """
+        if default is _sentinel:
+            return self.__dict__.pop(name)
+        else:
+            return self.__dict__.pop(name, default)
+
+    def setdefault(self, name, default=None):
+        """Get the value of an attribute if it is present, otherwise
+        set and return a default value. Like :meth:`dict.setdefault`.
+
+        :param name: Name of attribute to get.
+        :param: default: Value to set and return if the attribute is not
+            present.
+
+        .. versionadded:: 0.11
+        """
+        return self.__dict__.setdefault(name, default)
+
+    def __contains__(self, item):
+        return item in self.__dict__
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def __repr__(self):
+        top = _app_ctx_stack.top
+        if top is not None:
+            return '<flask.g of %r>' % top.app.name
+        return object.__repr__(self)
+
+
+def after_this_request(f):
+    """Executes a function after this request.  This is useful to modify
+    response objects.  The function is passed the response object and has
+    to return the same or a new one.
+
+    Example::
+
+        @app.route('/')
+        def index():
+            @after_this_request
+            def add_header(response):
+                response.headers['X-Foo'] = 'Parachute'
+                return response
+            return 'Hello World!'
+
+    This is more useful if a function other than the view function wants to
+    modify a response.  For instance think of a decorator that wants to add
+    some headers without converting the return value into a response object.
+
+    .. versionadded:: 0.9
+    """
+    _request_ctx_stack.top._after_request_functions.append(f)
+    return f
+
+
+def copy_current_request_context(f):
+    """A helper function that decorates a function to retain the current
+    request context.  This is useful when working with greenlets.  The moment
+    the function is decorated a copy of the request context is created and
+    then pushed when the function is called.
+
+    Example::
+
+        import gevent
+        from flask import copy_current_request_context
+
+        @app.route('/')
+        def index():
+            @copy_current_request_context
+            def do_some_work():
+                # do some work here, it can access flask.request like you
+                # would otherwise in the view function.
+                ...
+            gevent.spawn(do_some_work)
+            return 'Regular response'
+
+    .. versionadded:: 0.10
+    """
+    top = _request_ctx_stack.top
+    if top is None:
+        raise RuntimeError('This decorator can only be used at local scopes '
+            'when a request context is on the stack.  For instance within '
+            'view functions.')
+    reqctx = top.copy()
+    def wrapper(*args, **kwargs):
+        with reqctx:
+            return f(*args, **kwargs)
+    return update_wrapper(wrapper, f)
+
+
+def has_request_context():
+    """If you have code that wants to test if a request context is there or
+    not this function can be used.  For instance, you may want to take advantage
+    of request information if the request object is available, but fail
+    silently if it is unavailable.
+
+    ::
+
+        class User(db.Model):
+
+            def __init__(self, username, remote_addr=None):
+                self.username = username
+                if remote_addr is None and has_request_context():
+                    remote_addr = request.remote_addr
+                self.remote_addr = remote_addr
+
+    Alternatively you can also just test any of the context bound objects
+    (such as :class:`request` or :class:`g`) for truthness::
+
+        class User(db.Model):
+
+            def __init__(self, username, remote_addr=None):
+                self.username = username
+                if remote_addr is None and request:
+                    remote_addr = request.remote_addr
+                self.remote_addr = remote_addr
+
+    .. versionadded:: 0.7
+    """
+    return _request_ctx_stack.top is not None
+
+
+def has_app_context():
+    """Works like :func:`has_request_context` but for the application
+    context.  You can also just do a boolean check on the
+    :data:`current_app` object instead.
+
+    .. versionadded:: 0.9
+    """
+    return _app_ctx_stack.top is not None
+
+
+class AppContext(object):
+    """The application context binds an application object implicitly
+    to the current thread or greenlet, similar to how the
+    :class:`RequestContext` binds request information.  The application
+    context is also implicitly created if a request context is created
+    but the application is not on top of the individual application
+    context.
+    """
+
+    def __init__(self, app):
+        self.app = app
+        self.url_adapter = app.create_url_adapter(None)
+        self.g = app.app_ctx_globals_class()
+
+        # Like request context, app contexts can be pushed multiple times
+        # but there a basic "refcount" is enough to track them.
+        self._refcnt = 0
+
+    def push(self):
+        """Binds the app context to the current context."""
+        self._refcnt += 1
+        if hasattr(sys, 'exc_clear'):
+            sys.exc_clear()
+        _app_ctx_stack.push(self)
+        appcontext_pushed.send(self.app)
+
+    def pop(self, exc=_sentinel):
+        """Pops the app context."""
+        try:
+            self._refcnt -= 1
+            if self._refcnt <= 0:
+                if exc is _sentinel:
+                    exc = sys.exc_info()[1]
+                self.app.do_teardown_appcontext(exc)
+        finally:
+            rv = _app_ctx_stack.pop()
+        assert rv is self, 'Popped wrong app context.  (%r instead of %r)' \
+            % (rv, self)
+        appcontext_popped.send(self.app)
+
+    def __enter__(self):
+        self.push()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.pop(exc_value)
+
+        if BROKEN_PYPY_CTXMGR_EXIT and exc_type is not None:
+            reraise(exc_type, exc_value, tb)
+
+
+class RequestContext(object):
+    """The request context contains all request relevant information.  It is
+    created at the beginning of the request and pushed to the
+    `_request_ctx_stack` and removed at the end of it.  It will create the
+    URL adapter and request object for the WSGI environment provided.
+
+    Do not attempt to use this class directly, instead use
+    :meth:`~flask.Flask.test_request_context` and
+    :meth:`~flask.Flask.request_context` to create this object.
+
+    When the request context is popped, it will evaluate all the
+    functions registered on the application for teardown execution
+    (:meth:`~flask.Flask.teardown_request`).
+
+    The request context is automatically popped at the end of the request
+    for you.  In debug mode the request context is kept around if
+    exceptions happen so that interactive debuggers have a chance to
+    introspect the data.  With 0.4 this can also be forced for requests
+    that did not fail and outside of ``DEBUG`` mode.  By setting
+    ``'flask._preserve_context'`` to ``True`` on the WSGI environment the
+    context will not pop itself at the end of the request.  This is used by
+    the :meth:`~flask.Flask.test_client` for example to implement the
+    deferred cleanup functionality.
+
+    You might find this helpful for unittests where you need the
+    information from the context local around for a little longer.  Make
+    sure to properly :meth:`~werkzeug.LocalStack.pop` the stack yourself in
+    that situation, otherwise your unittests will leak memory.
+    """
+
+    def __init__(self, app, environ, request=None):
+        self.app = app
+        if request is None:
+            request = app.request_class(environ)
+        self.request = request
+        self.url_adapter = app.create_url_adapter(self.request)
+        self.flashes = None
+        self.session = None
+
+        # Request contexts can be pushed multiple times and interleaved with
+        # other request contexts.  Now only if the last level is popped we
+        # get rid of them.  Additionally if an application context is missing
+        # one is created implicitly so for each level we add this information
+        self._implicit_app_ctx_stack = []
+
+        # indicator if the context was preserved.  Next time another context
+        # is pushed the preserved context is popped.
+        self.preserved = False
+
+        # remembers the exception for pop if there is one in case the context
+        # preservation kicks in.
+        self._preserved_exc = None
+
+        # Functions that should be executed after the request on the response
+        # object.  These will be called before the regular "after_request"
+        # functions.
+        self._after_request_functions = []
+
+        self.match_request()
+
+    def _get_g(self):
+        return _app_ctx_stack.top.g
+    def _set_g(self, value):
+        _app_ctx_stack.top.g = value
+    g = property(_get_g, _set_g)
+    del _get_g, _set_g
+
+    def copy(self):
+        """Creates a copy of this request context with the same request object.
+        This can be used to move a request context to a different greenlet.
+        Because the actual request object is the same this cannot be used to
+        move a request context to a different thread unless access to the
+        request object is locked.
+
+        .. versionadded:: 0.10
+        """
+        return self.__class__(self.app,
+            environ=self.request.environ,
+            request=self.request
+        )
+
+    def match_request(self):
+        """Can be overridden by a subclass to hook into the matching
+        of the request.
+        """
+        try:
+            url_rule, self.request.view_args = \
+                self.url_adapter.match(return_rule=True)
+            self.request.url_rule = url_rule
+        except HTTPException as e:
+            self.request.routing_exception = e
+
+    def push(self):
+        """Binds the request context to the current context."""
+        # If an exception occurs in debug mode or if context preservation is
+        # activated under exception situations exactly one context stays
+        # on the stack.  The rationale is that you want to access that
+        # information under debug situations.  However if someone forgets to
+        # pop that context again we want to make sure that on the next push
+        # it's invalidated, otherwise we run at risk that something leaks
+        # memory.  This is usually only a problem in test suite since this
+        # functionality is not active in production environments.
+        top = _request_ctx_stack.top
+        if top is not None and top.preserved:
+            top.pop(top._preserved_exc)
+
+        # Before we push the request context we have to ensure that there
+        # is an application context.
+        app_ctx = _app_ctx_stack.top
+        if app_ctx is None or app_ctx.app != self.app:
+            app_ctx = self.app.app_context()
+            app_ctx.push()
+            self._implicit_app_ctx_stack.append(app_ctx)
+        else:
+            self._implicit_app_ctx_stack.append(None)
+
+        if hasattr(sys, 'exc_clear'):
+            sys.exc_clear()
+
+        _request_ctx_stack.push(self)
+
+        # Open the session at the moment that the request context is available.
+        # This allows a custom open_session method to use the request context.
+        # Only open a new session if this is the first time the request was
+        # pushed, otherwise stream_with_context loses the session.
+        if self.session is None:
+            session_interface = self.app.session_interface
+            self.session = session_interface.open_session(
+                self.app, self.request
+            )
+
+            if self.session is None:
+                self.session = session_interface.make_null_session(self.app)
+
+    def pop(self, exc=_sentinel):
+        """Pops the request context and unbinds it by doing that.  This will
+        also trigger the execution of functions registered by the
+        :meth:`~flask.Flask.teardown_request` decorator.
+
+        .. versionchanged:: 0.9
+           Added the `exc` argument.
+        """
+        app_ctx = self._implicit_app_ctx_stack.pop()
+
+        try:
+            clear_request = False
+            if not self._implicit_app_ctx_stack:
+                self.preserved = False
+                self._preserved_exc = None
+                if exc is _sentinel:
+                    exc = sys.exc_info()[1]
+                self.app.do_teardown_request(exc)
+
+                # If this interpreter supports clearing the exception information
+                # we do that now.  This will only go into effect on Python 2.x,
+                # on 3.x it disappears automatically at the end of the exception
+                # stack.
+                if hasattr(sys, 'exc_clear'):
+                    sys.exc_clear()
+
+                request_close = getattr(self.request, 'close', None)
+                if request_close is not None:
+                    request_close()
+                clear_request = True
+        finally:
+            rv = _request_ctx_stack.pop()
+
+            # get rid of circular dependencies at the end of the request
+            # so that we don't require the GC to be active.
+            if clear_request:
+                rv.request.environ['werkzeug.request'] = None
+
+            # Get rid of the app as well if necessary.
+            if app_ctx is not None:
+                app_ctx.pop(exc)
+
+            assert rv is self, 'Popped wrong request context.  ' \
+                '(%r instead of %r)' % (rv, self)
+
+    def auto_pop(self, exc):
+        if self.request.environ.get('flask._preserve_context') or \
+           (exc is not None and self.app.preserve_context_on_exception):
+            self.preserved = True
+            self._preserved_exc = exc
+        else:
+            self.pop(exc)
+
+    def __enter__(self):
+        self.push()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        # do not pop the request stack if we are in debug mode and an
+        # exception happened.  This will allow the debugger to still
+        # access the request object in the interactive shell.  Furthermore
+        # the context can be force kept alive for the test client.
+        # See flask.testing for how this works.
+        self.auto_pop(exc_value)
+
+        if BROKEN_PYPY_CTXMGR_EXIT and exc_type is not None:
+            reraise(exc_type, exc_value, tb)
+
+    def __repr__(self):
+        return '<%s \'%s\' [%s] of %s>' % (
+            self.__class__.__name__,
+            self.request.url,
+            self.request.method,
+            self.app.name,
+        )

python/flask/debughelpers.py

@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.debughelpers
+    ~~~~~~~~~~~~~~~~~~
+
+    Various helpers to make the development experience better.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+from warnings import warn
+
+from ._compat import implements_to_string, text_type
+from .app import Flask
+from .blueprints import Blueprint
+from .globals import _request_ctx_stack
+
+
+class UnexpectedUnicodeError(AssertionError, UnicodeError):
+    """Raised in places where we want some better error reporting for
+    unexpected unicode or binary data.
+    """
+
+
+@implements_to_string
+class DebugFilesKeyError(KeyError, AssertionError):
+    """Raised from request.files during debugging.  The idea is that it can
+    provide a better error message than just a generic KeyError/BadRequest.
+    """
+
+    def __init__(self, request, key):
+        form_matches = request.form.getlist(key)
+        buf = ['You tried to access the file "%s" in the request.files '
+               'dictionary but it does not exist.  The mimetype for the request '
+               'is "%s" instead of "multipart/form-data" which means that no '
+               'file contents were transmitted.  To fix this error you should '
+               'provide enctype="multipart/form-data" in your form.' %
+               (key, request.mimetype)]
+        if form_matches:
+            buf.append('\n\nThe browser instead transmitted some file names. '
+                       'This was submitted: %s' % ', '.join('"%s"' % x
+                            for x in form_matches))
+        self.msg = ''.join(buf)
+
+    def __str__(self):
+        return self.msg
+
+
+class FormDataRoutingRedirect(AssertionError):
+    """This exception is raised by Flask in debug mode if it detects a
+    redirect caused by the routing system when the request method is not
+    GET, HEAD or OPTIONS.  Reasoning: form data will be dropped.
+    """
+
+    def __init__(self, request):
+        exc = request.routing_exception
+        buf = ['A request was sent to this URL (%s) but a redirect was '
+               'issued automatically by the routing system to "%s".'
+               % (request.url, exc.new_url)]
+
+        # In case just a slash was appended we can be extra helpful
+        if request.base_url + '/' == exc.new_url.split('?')[0]:
+            buf.append('  The URL was defined with a trailing slash so '
+                       'Flask will automatically redirect to the URL '
+                       'with the trailing slash if it was accessed '
+                       'without one.')
+
+        buf.append('  Make sure to directly send your %s-request to this URL '
+                   'since we can\'t make browsers or HTTP clients redirect '
+                   'with form data reliably or without user interaction.' %
+                   request.method)
+        buf.append('\n\nNote: this exception is only raised in debug mode')
+        AssertionError.__init__(self, ''.join(buf).encode('utf-8'))
+
+
+def attach_enctype_error_multidict(request):
+    """Since Flask 0.8 we're monkeypatching the files object in case a
+    request is detected that does not use multipart form data but the files
+    object is accessed.
+    """
+    oldcls = request.files.__class__
+    class newcls(oldcls):
+        def __getitem__(self, key):
+            try:
+                return oldcls.__getitem__(self, key)
+            except KeyError:
+                if key not in request.form:
+                    raise
+                raise DebugFilesKeyError(request, key)
+    newcls.__name__ = oldcls.__name__
+    newcls.__module__ = oldcls.__module__
+    request.files.__class__ = newcls
+
+
+def _dump_loader_info(loader):
+    yield 'class: %s.%s' % (type(loader).__module__, type(loader).__name__)
+    for key, value in sorted(loader.__dict__.items()):
+        if key.startswith('_'):
+            continue
+        if isinstance(value, (tuple, list)):
+            if not all(isinstance(x, (str, text_type)) for x in value):
+                continue
+            yield '%s:' % key
+            for item in value:
+                yield '  - %s' % item
+            continue
+        elif not isinstance(value, (str, text_type, int, float, bool)):
+            continue
+        yield '%s: %r' % (key, value)
+
+
+def explain_template_loading_attempts(app, template, attempts):
+    """This should help developers understand what failed"""
+    info = ['Locating template "%s":' % template]
+    total_found = 0
+    blueprint = None
+    reqctx = _request_ctx_stack.top
+    if reqctx is not None and reqctx.request.blueprint is not None:
+        blueprint = reqctx.request.blueprint
+
+    for idx, (loader, srcobj, triple) in enumerate(attempts):
+        if isinstance(srcobj, Flask):
+            src_info = 'application "%s"' % srcobj.import_name
+        elif isinstance(srcobj, Blueprint):
+            src_info = 'blueprint "%s" (%s)' % (srcobj.name,
+                                                srcobj.import_name)
+        else:
+            src_info = repr(srcobj)
+
+        info.append('% 5d: trying loader of %s' % (
+            idx + 1, src_info))
+
+        for line in _dump_loader_info(loader):
+            info.append('       %s' % line)
+
+        if triple is None:
+            detail = 'no match'
+        else:
+            detail = 'found (%r)' % (triple[1] or '<string>')
+            total_found += 1
+        info.append('       -> %s' % detail)
+
+    seems_fishy = False
+    if total_found == 0:
+        info.append('Error: the template could not be found.')
+        seems_fishy = True
+    elif total_found > 1:
+        info.append('Warning: multiple loaders returned a match for the template.')
+        seems_fishy = True
+
+    if blueprint is not None and seems_fishy:
+        info.append('  The template was looked up from an endpoint that '
+                    'belongs to the blueprint "%s".' % blueprint)
+        info.append('  Maybe you did not place a template in the right folder?')
+        info.append('  See http://flask.pocoo.org/docs/blueprints/#templates')
+
+    app.logger.info('\n'.join(info))
+
+
+def explain_ignored_app_run():
+    if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
+        warn(Warning('Silently ignoring app.run() because the '
+                     'application is run from the flask command line '
+                     'executable.  Consider putting app.run() behind an '
+                     'if __name__ == "__main__" guard to silence this '
+                     'warning.'), stacklevel=3)

python/flask/globals.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.globals
+    ~~~~~~~~~~~~~
+
+    Defines all the global objects that are proxies to the current
+    active context.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from functools import partial
+from werkzeug.local import LocalStack, LocalProxy
+
+
+_request_ctx_err_msg = '''\
+Working outside of request context.
+
+This typically means that you attempted to use functionality that needed
+an active HTTP request.  Consult the documentation on testing for
+information about how to avoid this problem.\
+'''
+_app_ctx_err_msg = '''\
+Working outside of application context.
+
+This typically means that you attempted to use functionality that needed
+to interface with the current application object in some way. To solve
+this, set up an application context with app.app_context().  See the
+documentation for more information.\
+'''
+
+
+def _lookup_req_object(name):
+    top = _request_ctx_stack.top
+    if top is None:
+        raise RuntimeError(_request_ctx_err_msg)
+    return getattr(top, name)
+
+
+def _lookup_app_object(name):
+    top = _app_ctx_stack.top
+    if top is None:
+        raise RuntimeError(_app_ctx_err_msg)
+    return getattr(top, name)
+
+
+def _find_app():
+    top = _app_ctx_stack.top
+    if top is None:
+        raise RuntimeError(_app_ctx_err_msg)
+    return top.app
+
+
+# context locals
+_request_ctx_stack = LocalStack()
+_app_ctx_stack = LocalStack()
+current_app = LocalProxy(_find_app)
+request = LocalProxy(partial(_lookup_req_object, 'request'))
+session = LocalProxy(partial(_lookup_req_object, 'session'))
+g = LocalProxy(partial(_lookup_app_object, 'g'))

python/flask/json/__init__.py

@@ -0,0 +1,357 @@
+# -*- coding: utf-8 -*-
+"""
+flask.json
+~~~~~~~~~~
+
+:copyright: © 2010 by the Pallets team.
+:license: BSD, see LICENSE for more details.
+"""
+import codecs
+import io
+import uuid
+from datetime import date, datetime
+from flask.globals import current_app, request
+from flask._compat import text_type, PY2
+
+from werkzeug.http import http_date
+from jinja2 import Markup
+
+# Use the same json implementation as itsdangerous on which we
+# depend anyways.
+from itsdangerous import json as _json
+
+
+# Figure out if simplejson escapes slashes.  This behavior was changed
+# from one version to another without reason.
+_slash_escape = '\\/' not in _json.dumps('/')
+
+
+__all__ = ['dump', 'dumps', 'load', 'loads', 'htmlsafe_dump',
+           'htmlsafe_dumps', 'JSONDecoder', 'JSONEncoder',
+           'jsonify']
+
+
+def _wrap_reader_for_text(fp, encoding):
+    if isinstance(fp.read(0), bytes):
+        fp = io.TextIOWrapper(io.BufferedReader(fp), encoding)
+    return fp
+
+
+def _wrap_writer_for_text(fp, encoding):
+    try:
+        fp.write('')
+    except TypeError:
+        fp = io.TextIOWrapper(fp, encoding)
+    return fp
+
+
+class JSONEncoder(_json.JSONEncoder):
+    """The default Flask JSON encoder.  This one extends the default simplejson
+    encoder by also supporting ``datetime`` objects, ``UUID`` as well as
+    ``Markup`` objects which are serialized as RFC 822 datetime strings (same
+    as the HTTP date format).  In order to support more data types override the
+    :meth:`default` method.
+    """
+
+    def default(self, o):
+        """Implement this method in a subclass such that it returns a
+        serializable object for ``o``, or calls the base implementation (to
+        raise a :exc:`TypeError`).
+
+        For example, to support arbitrary iterators, you could implement
+        default like this::
+
+            def default(self, o):
+                try:
+                    iterable = iter(o)
+                except TypeError:
+                    pass
+                else:
+                    return list(iterable)
+                return JSONEncoder.default(self, o)
+        """
+        if isinstance(o, datetime):
+            return http_date(o.utctimetuple())
+        if isinstance(o, date):
+            return http_date(o.timetuple())
+        if isinstance(o, uuid.UUID):
+            return str(o)
+        if hasattr(o, '__html__'):
+            return text_type(o.__html__())
+        return _json.JSONEncoder.default(self, o)
+
+
+class JSONDecoder(_json.JSONDecoder):
+    """The default JSON decoder.  This one does not change the behavior from
+    the default simplejson decoder.  Consult the :mod:`json` documentation
+    for more information.  This decoder is not only used for the load
+    functions of this module but also :attr:`~flask.Request`.
+    """
+
+
+def _dump_arg_defaults(kwargs, app=None):
+    """Inject default arguments for dump functions."""
+    if app is None:
+        app = current_app
+
+    if app:
+        bp = app.blueprints.get(request.blueprint) if request else None
+        kwargs.setdefault(
+            'cls', bp.json_encoder if bp and bp.json_encoder else app.json_encoder
+        )
+
+        if not app.config['JSON_AS_ASCII']:
+            kwargs.setdefault('ensure_ascii', False)
+
+        kwargs.setdefault('sort_keys', app.config['JSON_SORT_KEYS'])
+    else:
+        kwargs.setdefault('sort_keys', True)
+        kwargs.setdefault('cls', JSONEncoder)
+
+
+def _load_arg_defaults(kwargs, app=None):
+    """Inject default arguments for load functions."""
+    if app is None:
+        app = current_app
+
+    if app:
+        bp = app.blueprints.get(request.blueprint) if request else None
+        kwargs.setdefault(
+            'cls',
+            bp.json_decoder if bp and bp.json_decoder
+                else app.json_decoder
+        )
+    else:
+        kwargs.setdefault('cls', JSONDecoder)
+
+
+def detect_encoding(data):
+    """Detect which UTF codec was used to encode the given bytes.
+
+    The latest JSON standard (:rfc:`8259`) suggests that only UTF-8 is
+    accepted. Older documents allowed 8, 16, or 32. 16 and 32 can be big
+    or little endian. Some editors or libraries may prepend a BOM.
+
+    :param data: Bytes in unknown UTF encoding.
+    :return: UTF encoding name
+    """
+    head = data[:4]
+
+    if head[:3] == codecs.BOM_UTF8:
+        return 'utf-8-sig'
+
+    if b'\x00' not in head:
+        return 'utf-8'
+
+    if head in (codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE):
+        return 'utf-32'
+
+    if head[:2] in (codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE):
+        return 'utf-16'
+
+    if len(head) == 4:
+        if head[:3] == b'\x00\x00\x00':
+            return 'utf-32-be'
+
+        if head[::2] == b'\x00\x00':
+            return 'utf-16-be'
+
+        if head[1:] == b'\x00\x00\x00':
+            return 'utf-32-le'
+
+        if head[1::2] == b'\x00\x00':
+            return 'utf-16-le'
+
+    if len(head) == 2:
+        return 'utf-16-be' if head.startswith(b'\x00') else 'utf-16-le'
+
+    return 'utf-8'
+
+
+def dumps(obj, app=None, **kwargs):
+    """Serialize ``obj`` to a JSON-formatted string. If there is an
+    app context pushed, use the current app's configured encoder
+    (:attr:`~flask.Flask.json_encoder`), or fall back to the default
+    :class:`JSONEncoder`.
+
+    Takes the same arguments as the built-in :func:`json.dumps`, and
+    does some extra configuration based on the application. If the
+    simplejson package is installed, it is preferred.
+
+    :param obj: Object to serialize to JSON.
+    :param app: App instance to use to configure the JSON encoder.
+        Uses ``current_app`` if not given, and falls back to the default
+        encoder when not in an app context.
+    :param kwargs: Extra arguments passed to :func:`json.dumps`.
+
+    .. versionchanged:: 1.0.3
+
+        ``app`` can be passed directly, rather than requiring an app
+        context for configuration.
+    """
+    _dump_arg_defaults(kwargs, app=app)
+    encoding = kwargs.pop('encoding', None)
+    rv = _json.dumps(obj, **kwargs)
+    if encoding is not None and isinstance(rv, text_type):
+        rv = rv.encode(encoding)
+    return rv
+
+
+def dump(obj, fp, app=None, **kwargs):
+    """Like :func:`dumps` but writes into a file object."""
+    _dump_arg_defaults(kwargs, app=app)
+    encoding = kwargs.pop('encoding', None)
+    if encoding is not None:
+        fp = _wrap_writer_for_text(fp, encoding)
+    _json.dump(obj, fp, **kwargs)
+
+
+def loads(s, app=None, **kwargs):
+    """Deserialize an object from a JSON-formatted string ``s``. If
+    there is an app context pushed, use the current app's configured
+    decoder (:attr:`~flask.Flask.json_decoder`), or fall back to the
+    default :class:`JSONDecoder`.
+
+    Takes the same arguments as the built-in :func:`json.loads`, and
+    does some extra configuration based on the application. If the
+    simplejson package is installed, it is preferred.
+
+    :param s: JSON string to deserialize.
+    :param app: App instance to use to configure the JSON decoder.
+        Uses ``current_app`` if not given, and falls back to the default
+        encoder when not in an app context.
+    :param kwargs: Extra arguments passed to :func:`json.dumps`.
+
+    .. versionchanged:: 1.0.3
+
+        ``app`` can be passed directly, rather than requiring an app
+        context for configuration.
+    """
+    _load_arg_defaults(kwargs, app=app)
+    if isinstance(s, bytes):
+        encoding = kwargs.pop('encoding', None)
+        if encoding is None:
+            encoding = detect_encoding(s)
+        s = s.decode(encoding)
+    return _json.loads(s, **kwargs)
+
+
+def load(fp, app=None, **kwargs):
+    """Like :func:`loads` but reads from a file object."""
+    _load_arg_defaults(kwargs, app=app)
+    if not PY2:
+        fp = _wrap_reader_for_text(fp, kwargs.pop('encoding', None) or 'utf-8')
+    return _json.load(fp, **kwargs)
+
+
+def htmlsafe_dumps(obj, **kwargs):
+    """Works exactly like :func:`dumps` but is safe for use in ``<script>``
+    tags.  It accepts the same arguments and returns a JSON string.  Note that
+    this is available in templates through the ``|tojson`` filter which will
+    also mark the result as safe.  Due to how this function escapes certain
+    characters this is safe even if used outside of ``<script>`` tags.
+
+    The following characters are escaped in strings:
+
+    -   ``<``
+    -   ``>``
+    -   ``&``
+    -   ``'``
+
+    This makes it safe to embed such strings in any place in HTML with the
+    notable exception of double quoted attributes.  In that case single
+    quote your attributes or HTML escape it in addition.
+
+    .. versionchanged:: 0.10
+       This function's return value is now always safe for HTML usage, even
+       if outside of script tags or if used in XHTML.  This rule does not
+       hold true when using this function in HTML attributes that are double
+       quoted.  Always single quote attributes if you use the ``|tojson``
+       filter.  Alternatively use ``|tojson|forceescape``.
+    """
+    rv = dumps(obj, **kwargs) \
+        .replace(u'<', u'\\u003c') \
+        .replace(u'>', u'\\u003e') \
+        .replace(u'&', u'\\u0026') \
+        .replace(u"'", u'\\u0027')
+    if not _slash_escape:
+        rv = rv.replace('\\/', '/')
+    return rv
+
+
+def htmlsafe_dump(obj, fp, **kwargs):
+    """Like :func:`htmlsafe_dumps` but writes into a file object."""
+    fp.write(text_type(htmlsafe_dumps(obj, **kwargs)))
+
+
+def jsonify(*args, **kwargs):
+    """This function wraps :func:`dumps` to add a few enhancements that make
+    life easier.  It turns the JSON output into a :class:`~flask.Response`
+    object with the :mimetype:`application/json` mimetype.  For convenience, it
+    also converts multiple arguments into an array or multiple keyword arguments
+    into a dict.  This means that both ``jsonify(1,2,3)`` and
+    ``jsonify([1,2,3])`` serialize to ``[1,2,3]``.
+
+    For clarity, the JSON serialization behavior has the following differences
+    from :func:`dumps`:
+
+    1. Single argument: Passed straight through to :func:`dumps`.
+    2. Multiple arguments: Converted to an array before being passed to
+       :func:`dumps`.
+    3. Multiple keyword arguments: Converted to a dict before being passed to
+       :func:`dumps`.
+    4. Both args and kwargs: Behavior undefined and will throw an exception.
+
+    Example usage::
+
+        from flask import jsonify
+
+        @app.route('/_get_current_user')
+        def get_current_user():
+            return jsonify(username=g.user.username,
+                           email=g.user.email,
+                           id=g.user.id)
+
+    This will send a JSON response like this to the browser::
+
+        {
+            "username": "admin",
+            "email": "admin@localhost",
+            "id": 42
+        }
+
+
+    .. versionchanged:: 0.11
+       Added support for serializing top-level arrays. This introduces a
+       security risk in ancient browsers. See :ref:`json-security` for details.
+
+    This function's response will be pretty printed if the
+    ``JSONIFY_PRETTYPRINT_REGULAR`` config parameter is set to True or the
+    Flask app is running in debug mode. Compressed (not pretty) formatting
+    currently means no indents and no spaces after separators.
+
+    .. versionadded:: 0.2
+    """
+
+    indent = None
+    separators = (',', ':')
+
+    if current_app.config['JSONIFY_PRETTYPRINT_REGULAR'] or current_app.debug:
+        indent = 2
+        separators = (', ', ': ')
+
+    if args and kwargs:
+        raise TypeError('jsonify() behavior undefined when passed both args and kwargs')
+    elif len(args) == 1:  # single args are passed directly to dumps()
+        data = args[0]
+    else:
+        data = args or kwargs
+
+    return current_app.response_class(
+        dumps(data, indent=indent, separators=separators) + '\n',
+        mimetype=current_app.config['JSONIFY_MIMETYPE']
+    )
+
+
+def tojson_filter(obj, **kwargs):
+    return Markup(htmlsafe_dumps(obj, **kwargs))

python/flask/json/tag.py

@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+"""
+Tagged JSON
+~~~~~~~~~~~
+
+A compact representation for lossless serialization of non-standard JSON types.
+:class:`~flask.sessions.SecureCookieSessionInterface` uses this to serialize
+the session data, but it may be useful in other places. It can be extended to
+support other types.
+
+.. autoclass:: TaggedJSONSerializer
+    :members:
+
+.. autoclass:: JSONTag
+    :members:
+
+Let's seen an example that adds support for :class:`~collections.OrderedDict`.
+Dicts don't have an order in Python or JSON, so to handle this we will dump
+the items as a list of ``[key, value]`` pairs. Subclass :class:`JSONTag` and
+give it the new key ``' od'`` to identify the type. The session serializer
+processes dicts first, so insert the new tag at the front of the order since
+``OrderedDict`` must be processed before ``dict``. ::
+
+    from flask.json.tag import JSONTag
+
+    class TagOrderedDict(JSONTag):
+        __slots__ = ('serializer',)
+        key = ' od'
+
+        def check(self, value):
+            return isinstance(value, OrderedDict)
+
+        def to_json(self, value):
+            return [[k, self.serializer.tag(v)] for k, v in iteritems(value)]
+
+        def to_python(self, value):
+            return OrderedDict(value)
+
+    app.session_interface.serializer.register(TagOrderedDict, index=0)
+
+:copyright: © 2010 by the Pallets team.
+:license: BSD, see LICENSE for more details.
+"""
+
+from base64 import b64decode, b64encode
+from datetime import datetime
+from uuid import UUID
+
+from jinja2 import Markup
+from werkzeug.http import http_date, parse_date
+
+from flask._compat import iteritems, text_type
+from flask.json import dumps, loads
+
+
+class JSONTag(object):
+    """Base class for defining type tags for :class:`TaggedJSONSerializer`."""
+
+    __slots__ = ('serializer',)
+
+    #: The tag to mark the serialized object with. If ``None``, this tag is
+    #: only used as an intermediate step during tagging.
+    key = None
+
+    def __init__(self, serializer):
+        """Create a tagger for the given serializer."""
+        self.serializer = serializer
+
+    def check(self, value):
+        """Check if the given value should be tagged by this tag."""
+        raise NotImplementedError
+
+    def to_json(self, value):
+        """Convert the Python object to an object that is a valid JSON type.
+        The tag will be added later."""
+        raise NotImplementedError
+
+    def to_python(self, value):
+        """Convert the JSON representation back to the correct type. The tag
+        will already be removed."""
+        raise NotImplementedError
+
+    def tag(self, value):
+        """Convert the value to a valid JSON type and add the tag structure
+        around it."""
+        return {self.key: self.to_json(value)}
+
+
+class TagDict(JSONTag):
+    """Tag for 1-item dicts whose only key matches a registered tag.
+
+    Internally, the dict key is suffixed with `__`, and the suffix is removed
+    when deserializing.
+    """
+
+    __slots__ = ()
+    key = ' di'
+
+    def check(self, value):
+        return (
+            isinstance(value, dict)
+            and len(value) == 1
+            and next(iter(value)) in self.serializer.tags
+        )
+
+    def to_json(self, value):
+        key = next(iter(value))
+        return {key + '__': self.serializer.tag(value[key])}
+
+    def to_python(self, value):
+        key = next(iter(value))
+        return {key[:-2]: value[key]}
+
+
+class PassDict(JSONTag):
+    __slots__ = ()
+
+    def check(self, value):
+        return isinstance(value, dict)
+
+    def to_json(self, value):
+        # JSON objects may only have string keys, so don't bother tagging the
+        # key here.
+        return dict((k, self.serializer.tag(v)) for k, v in iteritems(value))
+
+    tag = to_json
+
+
+class TagTuple(JSONTag):
+    __slots__ = ()
+    key = ' t'
+
+    def check(self, value):
+        return isinstance(value, tuple)
+
+    def to_json(self, value):
+        return [self.serializer.tag(item) for item in value]
+
+    def to_python(self, value):
+        return tuple(value)
+
+
+class PassList(JSONTag):
+    __slots__ = ()
+
+    def check(self, value):
+        return isinstance(value, list)
+
+    def to_json(self, value):
+        return [self.serializer.tag(item) for item in value]
+
+    tag = to_json
+
+
+class TagBytes(JSONTag):
+    __slots__ = ()
+    key = ' b'
+
+    def check(self, value):
+        return isinstance(value, bytes)
+
+    def to_json(self, value):
+        return b64encode(value).decode('ascii')
+
+    def to_python(self, value):
+        return b64decode(value)
+
+
+class TagMarkup(JSONTag):
+    """Serialize anything matching the :class:`~flask.Markup` API by
+    having a ``__html__`` method to the result of that method. Always
+    deserializes to an instance of :class:`~flask.Markup`."""
+
+    __slots__ = ()
+    key = ' m'
+
+    def check(self, value):
+        return callable(getattr(value, '__html__', None))
+
+    def to_json(self, value):
+        return text_type(value.__html__())
+
+    def to_python(self, value):
+        return Markup(value)
+
+
+class TagUUID(JSONTag):
+    __slots__ = ()
+    key = ' u'
+
+    def check(self, value):
+        return isinstance(value, UUID)
+
+    def to_json(self, value):
+        return value.hex
+
+    def to_python(self, value):
+        return UUID(value)
+
+
+class TagDateTime(JSONTag):
+    __slots__ = ()
+    key = ' d'
+
+    def check(self, value):
+        return isinstance(value, datetime)
+
+    def to_json(self, value):
+        return http_date(value)
+
+    def to_python(self, value):
+        return parse_date(value)
+
+
+class TaggedJSONSerializer(object):
+    """Serializer that uses a tag system to compactly represent objects that
+    are not JSON types. Passed as the intermediate serializer to
+    :class:`itsdangerous.Serializer`.
+
+    The following extra types are supported:
+
+    * :class:`dict`
+    * :class:`tuple`
+    * :class:`bytes`
+    * :class:`~flask.Markup`
+    * :class:`~uuid.UUID`
+    * :class:`~datetime.datetime`
+    """
+
+    __slots__ = ('tags', 'order')
+
+    #: Tag classes to bind when creating the serializer. Other tags can be
+    #: added later using :meth:`~register`.
+    default_tags = [
+        TagDict, PassDict, TagTuple, PassList, TagBytes, TagMarkup, TagUUID,
+        TagDateTime,
+    ]
+
+    def __init__(self):
+        self.tags = {}
+        self.order = []
+
+        for cls in self.default_tags:
+            self.register(cls)
+
+    def register(self, tag_class, force=False, index=None):
+        """Register a new tag with this serializer.
+
+        :param tag_class: tag class to register. Will be instantiated with this
+            serializer instance.
+        :param force: overwrite an existing tag. If false (default), a
+            :exc:`KeyError` is raised.
+        :param index: index to insert the new tag in the tag order. Useful when
+            the new tag is a special case of an existing tag. If ``None``
+            (default), the tag is appended to the end of the order.
+
+        :raise KeyError: if the tag key is already registered and ``force`` is
+            not true.
+        """
+        tag = tag_class(self)
+        key = tag.key
+
+        if key is not None:
+            if not force and key in self.tags:
+                raise KeyError("Tag '{0}' is already registered.".format(key))
+
+            self.tags[key] = tag
+
+        if index is None:
+            self.order.append(tag)
+        else:
+            self.order.insert(index, tag)
+
+    def tag(self, value):
+        """Convert a value to a tagged representation if necessary."""
+        for tag in self.order:
+            if tag.check(value):
+                return tag.tag(value)
+
+        return value
+
+    def untag(self, value):
+        """Convert a tagged representation back to the original type."""
+        if len(value) != 1:
+            return value
+
+        key = next(iter(value))
+
+        if key not in self.tags:
+            return value
+
+        return self.tags[key].to_python(value[key])
+
+    def dumps(self, value):
+        """Tag the value and dump it to a compact JSON string."""
+        return dumps(self.tag(value), separators=(',', ':'))
+
+    def loads(self, value):
+        """Load data from a JSON string and deserialized any tagged objects."""
+        return loads(value, object_hook=self.untag)

python/flask/logging.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+flask.logging
+~~~~~~~~~~~~~
+
+:copyright: © 2010 by the Pallets team.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import
+
+import logging
+import sys
+
+from werkzeug.local import LocalProxy
+
+from .globals import request
+
+
+@LocalProxy
+def wsgi_errors_stream():
+    """Find the most appropriate error stream for the application. If a request
+    is active, log to ``wsgi.errors``, otherwise use ``sys.stderr``.
+
+    If you configure your own :class:`logging.StreamHandler`, you may want to
+    use this for the stream. If you are using file or dict configuration and
+    can't import this directly, you can refer to it as
+    ``ext://flask.logging.wsgi_errors_stream``.
+    """
+    return request.environ['wsgi.errors'] if request else sys.stderr
+
+
+def has_level_handler(logger):
+    """Check if there is a handler in the logging chain that will handle the
+    given logger's :meth:`effective level <~logging.Logger.getEffectiveLevel>`.
+    """
+    level = logger.getEffectiveLevel()
+    current = logger
+
+    while current:
+        if any(handler.level <= level for handler in current.handlers):
+            return True
+
+        if not current.propagate:
+            break
+
+        current = current.parent
+
+    return False
+
+
+#: Log messages to :func:`~flask.logging.wsgi_errors_stream` with the format
+#: ``[%(asctime)s] %(levelname)s in %(module)s: %(message)s``.
+default_handler = logging.StreamHandler(wsgi_errors_stream)
+default_handler.setFormatter(logging.Formatter(
+    '[%(asctime)s] %(levelname)s in %(module)s: %(message)s'
+))
+
+
+def create_logger(app):
+    """Get the ``'flask.app'`` logger and configure it if needed.
+
+    When :attr:`~flask.Flask.debug` is enabled, set the logger level to
+    :data:`logging.DEBUG` if it is not set.
+
+    If there is no handler for the logger's effective level, add a
+    :class:`~logging.StreamHandler` for
+    :func:`~flask.logging.wsgi_errors_stream` with a basic format.
+    """
+    logger = logging.getLogger('flask.app')
+
+    if app.debug and logger.level == logging.NOTSET:
+        logger.setLevel(logging.DEBUG)
+
+    if not has_level_handler(logger):
+        logger.addHandler(default_handler)
+
+    return logger

python/flask/sessions.py

@@ -0,0 +1,385 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.sessions
+    ~~~~~~~~~~~~~~
+
+    Implements cookie based sessions based on itsdangerous.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import hashlib
+import warnings
+from datetime import datetime
+
+from itsdangerous import BadSignature, URLSafeTimedSerializer
+from werkzeug.datastructures import CallbackDict
+
+from flask._compat import collections_abc
+from flask.helpers import is_ip, total_seconds
+from flask.json.tag import TaggedJSONSerializer
+
+
+class SessionMixin(collections_abc.MutableMapping):
+    """Expands a basic dictionary with session attributes."""
+
+    @property
+    def permanent(self):
+        """This reflects the ``'_permanent'`` key in the dict."""
+        return self.get('_permanent', False)
+
+    @permanent.setter
+    def permanent(self, value):
+        self['_permanent'] = bool(value)
+
+    #: Some implementations can detect whether a session is newly
+    #: created, but that is not guaranteed. Use with caution. The mixin
+    # default is hard-coded ``False``.
+    new = False
+
+    #: Some implementations can detect changes to the session and set
+    #: this when that happens. The mixin default is hard coded to
+    #: ``True``.
+    modified = True
+
+    #: Some implementations can detect when session data is read or
+    #: written and set this when that happens. The mixin default is hard
+    #: coded to ``True``.
+    accessed = True
+
+
+class SecureCookieSession(CallbackDict, SessionMixin):
+    """Base class for sessions based on signed cookies.
+
+    This session backend will set the :attr:`modified` and
+    :attr:`accessed` attributes. It cannot reliably track whether a
+    session is new (vs. empty), so :attr:`new` remains hard coded to
+    ``False``.
+    """
+
+    #: When data is changed, this is set to ``True``. Only the session
+    #: dictionary itself is tracked; if the session contains mutable
+    #: data (for example a nested dict) then this must be set to
+    #: ``True`` manually when modifying that data. The session cookie
+    #: will only be written to the response if this is ``True``.
+    modified = False
+
+    #: When data is read or written, this is set to ``True``. Used by
+    # :class:`.SecureCookieSessionInterface` to add a ``Vary: Cookie``
+    #: header, which allows caching proxies to cache different pages for
+    #: different users.
+    accessed = False
+
+    def __init__(self, initial=None):
+        def on_update(self):
+            self.modified = True
+            self.accessed = True
+
+        super(SecureCookieSession, self).__init__(initial, on_update)
+
+    def __getitem__(self, key):
+        self.accessed = True
+        return super(SecureCookieSession, self).__getitem__(key)
+
+    def get(self, key, default=None):
+        self.accessed = True
+        return super(SecureCookieSession, self).get(key, default)
+
+    def setdefault(self, key, default=None):
+        self.accessed = True
+        return super(SecureCookieSession, self).setdefault(key, default)
+
+
+class NullSession(SecureCookieSession):
+    """Class used to generate nicer error messages if sessions are not
+    available.  Will still allow read-only access to the empty session
+    but fail on setting.
+    """
+
+    def _fail(self, *args, **kwargs):
+        raise RuntimeError('The session is unavailable because no secret '
+                           'key was set.  Set the secret_key on the '
+                           'application to something unique and secret.')
+    __setitem__ = __delitem__ = clear = pop = popitem = \
+        update = setdefault = _fail
+    del _fail
+
+
+class SessionInterface(object):
+    """The basic interface you have to implement in order to replace the
+    default session interface which uses werkzeug's securecookie
+    implementation.  The only methods you have to implement are
+    :meth:`open_session` and :meth:`save_session`, the others have
+    useful defaults which you don't need to change.
+
+    The session object returned by the :meth:`open_session` method has to
+    provide a dictionary like interface plus the properties and methods
+    from the :class:`SessionMixin`.  We recommend just subclassing a dict
+    and adding that mixin::
+
+        class Session(dict, SessionMixin):
+            pass
+
+    If :meth:`open_session` returns ``None`` Flask will call into
+    :meth:`make_null_session` to create a session that acts as replacement
+    if the session support cannot work because some requirement is not
+    fulfilled.  The default :class:`NullSession` class that is created
+    will complain that the secret key was not set.
+
+    To replace the session interface on an application all you have to do
+    is to assign :attr:`flask.Flask.session_interface`::
+
+        app = Flask(__name__)
+        app.session_interface = MySessionInterface()
+
+    .. versionadded:: 0.8
+    """
+
+    #: :meth:`make_null_session` will look here for the class that should
+    #: be created when a null session is requested.  Likewise the
+    #: :meth:`is_null_session` method will perform a typecheck against
+    #: this type.
+    null_session_class = NullSession
+
+    #: A flag that indicates if the session interface is pickle based.
+    #: This can be used by Flask extensions to make a decision in regards
+    #: to how to deal with the session object.
+    #:
+    #: .. versionadded:: 0.10
+    pickle_based = False
+
+    def make_null_session(self, app):
+        """Creates a null session which acts as a replacement object if the
+        real session support could not be loaded due to a configuration
+        error.  This mainly aids the user experience because the job of the
+        null session is to still support lookup without complaining but
+        modifications are answered with a helpful error message of what
+        failed.
+
+        This creates an instance of :attr:`null_session_class` by default.
+        """
+        return self.null_session_class()
+
+    def is_null_session(self, obj):
+        """Checks if a given object is a null session.  Null sessions are
+        not asked to be saved.
+
+        This checks if the object is an instance of :attr:`null_session_class`
+        by default.
+        """
+        return isinstance(obj, self.null_session_class)
+
+    def get_cookie_domain(self, app):
+        """Returns the domain that should be set for the session cookie.
+
+        Uses ``SESSION_COOKIE_DOMAIN`` if it is configured, otherwise
+        falls back to detecting the domain based on ``SERVER_NAME``.
+
+        Once detected (or if not set at all), ``SESSION_COOKIE_DOMAIN`` is
+        updated to avoid re-running the logic.
+        """
+
+        rv = app.config['SESSION_COOKIE_DOMAIN']
+
+        # set explicitly, or cached from SERVER_NAME detection
+        # if False, return None
+        if rv is not None:
+            return rv if rv else None
+
+        rv = app.config['SERVER_NAME']
+
+        # server name not set, cache False to return none next time
+        if not rv:
+            app.config['SESSION_COOKIE_DOMAIN'] = False
+            return None
+
+        # chop off the port which is usually not supported by browsers
+        # remove any leading '.' since we'll add that later
+        rv = rv.rsplit(':', 1)[0].lstrip('.')
+
+        if '.' not in rv:
+            # Chrome doesn't allow names without a '.'
+            # this should only come up with localhost
+            # hack around this by not setting the name, and show a warning
+            warnings.warn(
+                '"{rv}" is not a valid cookie domain, it must contain a ".".'
+                ' Add an entry to your hosts file, for example'
+                ' "{rv}.localdomain", and use that instead.'.format(rv=rv)
+            )
+            app.config['SESSION_COOKIE_DOMAIN'] = False
+            return None
+
+        ip = is_ip(rv)
+
+        if ip:
+            warnings.warn(
+                'The session cookie domain is an IP address. This may not work'
+                ' as intended in some browsers. Add an entry to your hosts'
+                ' file, for example "localhost.localdomain", and use that'
+                ' instead.'
+            )
+
+        # if this is not an ip and app is mounted at the root, allow subdomain
+        # matching by adding a '.' prefix
+        if self.get_cookie_path(app) == '/' and not ip:
+            rv = '.' + rv
+
+        app.config['SESSION_COOKIE_DOMAIN'] = rv
+        return rv
+
+    def get_cookie_path(self, app):
+        """Returns the path for which the cookie should be valid.  The
+        default implementation uses the value from the ``SESSION_COOKIE_PATH``
+        config var if it's set, and falls back to ``APPLICATION_ROOT`` or
+        uses ``/`` if it's ``None``.
+        """
+        return app.config['SESSION_COOKIE_PATH'] \
+               or app.config['APPLICATION_ROOT']
+
+    def get_cookie_httponly(self, app):
+        """Returns True if the session cookie should be httponly.  This
+        currently just returns the value of the ``SESSION_COOKIE_HTTPONLY``
+        config var.
+        """
+        return app.config['SESSION_COOKIE_HTTPONLY']
+
+    def get_cookie_secure(self, app):
+        """Returns True if the cookie should be secure.  This currently
+        just returns the value of the ``SESSION_COOKIE_SECURE`` setting.
+        """
+        return app.config['SESSION_COOKIE_SECURE']
+
+    def get_cookie_samesite(self, app):
+        """Return ``'Strict'`` or ``'Lax'`` if the cookie should use the
+        ``SameSite`` attribute. This currently just returns the value of
+        the :data:`SESSION_COOKIE_SAMESITE` setting.
+        """
+        return app.config['SESSION_COOKIE_SAMESITE']
+
+    def get_expiration_time(self, app, session):
+        """A helper method that returns an expiration date for the session
+        or ``None`` if the session is linked to the browser session.  The
+        default implementation returns now + the permanent session
+        lifetime configured on the application.
+        """
+        if session.permanent:
+            return datetime.utcnow() + app.permanent_session_lifetime
+
+    def should_set_cookie(self, app, session):
+        """Used by session backends to determine if a ``Set-Cookie`` header
+        should be set for this session cookie for this response. If the session
+        has been modified, the cookie is set. If the session is permanent and
+        the ``SESSION_REFRESH_EACH_REQUEST`` config is true, the cookie is
+        always set.
+
+        This check is usually skipped if the session was deleted.
+
+        .. versionadded:: 0.11
+        """
+
+        return session.modified or (
+            session.permanent and app.config['SESSION_REFRESH_EACH_REQUEST']
+        )
+
+    def open_session(self, app, request):
+        """This method has to be implemented and must either return ``None``
+        in case the loading failed because of a configuration error or an
+        instance of a session object which implements a dictionary like
+        interface + the methods and attributes on :class:`SessionMixin`.
+        """
+        raise NotImplementedError()
+
+    def save_session(self, app, session, response):
+        """This is called for actual sessions returned by :meth:`open_session`
+        at the end of the request.  This is still called during a request
+        context so if you absolutely need access to the request you can do
+        that.
+        """
+        raise NotImplementedError()
+
+
+session_json_serializer = TaggedJSONSerializer()
+
+
+class SecureCookieSessionInterface(SessionInterface):
+    """The default session interface that stores sessions in signed cookies
+    through the :mod:`itsdangerous` module.
+    """
+    #: the salt that should be applied on top of the secret key for the
+    #: signing of cookie based sessions.
+    salt = 'cookie-session'
+    #: the hash function to use for the signature.  The default is sha1
+    digest_method = staticmethod(hashlib.sha1)
+    #: the name of the itsdangerous supported key derivation.  The default
+    #: is hmac.
+    key_derivation = 'hmac'
+    #: A python serializer for the payload.  The default is a compact
+    #: JSON derived serializer with support for some extra Python types
+    #: such as datetime objects or tuples.
+    serializer = session_json_serializer
+    session_class = SecureCookieSession
+
+    def get_signing_serializer(self, app):
+        if not app.secret_key:
+            return None
+        signer_kwargs = dict(
+            key_derivation=self.key_derivation,
+            digest_method=self.digest_method
+        )
+        return URLSafeTimedSerializer(app.secret_key, salt=self.salt,
+                                      serializer=self.serializer,
+                                      signer_kwargs=signer_kwargs)
+
+    def open_session(self, app, request):
+        s = self.get_signing_serializer(app)
+        if s is None:
+            return None
+        val = request.cookies.get(app.session_cookie_name)
+        if not val:
+            return self.session_class()
+        max_age = total_seconds(app.permanent_session_lifetime)
+        try:
+            data = s.loads(val, max_age=max_age)
+            return self.session_class(data)
+        except BadSignature:
+            return self.session_class()
+
+    def save_session(self, app, session, response):
+        domain = self.get_cookie_domain(app)
+        path = self.get_cookie_path(app)
+
+        # If the session is modified to be empty, remove the cookie.
+        # If the session is empty, return without setting the cookie.
+        if not session:
+            if session.modified:
+                response.delete_cookie(
+                    app.session_cookie_name,
+                    domain=domain,
+                    path=path
+                )
+
+            return
+
+        # Add a "Vary: Cookie" header if the session was accessed at all.
+        if session.accessed:
+            response.vary.add('Cookie')
+
+        if not self.should_set_cookie(app, session):
+            return
+
+        httponly = self.get_cookie_httponly(app)
+        secure = self.get_cookie_secure(app)
+        samesite = self.get_cookie_samesite(app)
+        expires = self.get_expiration_time(app, session)
+        val = self.get_signing_serializer(app).dumps(dict(session))
+        response.set_cookie(
+            app.session_cookie_name,
+            val,
+            expires=expires,
+            httponly=httponly,
+            domain=domain,
+            path=path,
+            secure=secure,
+            samesite=samesite
+        )

python/flask/signals.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.signals
+    ~~~~~~~~~~~~~
+
+    Implements signals based on blinker if available, otherwise
+    falls silently back to a noop.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+signals_available = False
+try:
+    from blinker import Namespace
+    signals_available = True
+except ImportError:
+    class Namespace(object):
+        def signal(self, name, doc=None):
+            return _FakeSignal(name, doc)
+
+    class _FakeSignal(object):
+        """If blinker is unavailable, create a fake class with the same
+        interface that allows sending of signals but will fail with an
+        error on anything else.  Instead of doing anything on send, it
+        will just ignore the arguments and do nothing instead.
+        """
+
+        def __init__(self, name, doc=None):
+            self.name = name
+            self.__doc__ = doc
+        def _fail(self, *args, **kwargs):
+            raise RuntimeError('signalling support is unavailable '
+                               'because the blinker library is '
+                               'not installed.')
+        send = lambda *a, **kw: None
+        connect = disconnect = has_receivers_for = receivers_for = \
+            temporarily_connected_to = connected_to = _fail
+        del _fail
+
+# The namespace for code signals.  If you are not Flask code, do
+# not put signals in here.  Create your own namespace instead.
+_signals = Namespace()
+
+
+# Core signals.  For usage examples grep the source code or consult
+# the API documentation in docs/api.rst as well as docs/signals.rst
+template_rendered = _signals.signal('template-rendered')
+before_render_template = _signals.signal('before-render-template')
+request_started = _signals.signal('request-started')
+request_finished = _signals.signal('request-finished')
+request_tearing_down = _signals.signal('request-tearing-down')
+got_request_exception = _signals.signal('got-request-exception')
+appcontext_tearing_down = _signals.signal('appcontext-tearing-down')
+appcontext_pushed = _signals.signal('appcontext-pushed')
+appcontext_popped = _signals.signal('appcontext-popped')
+message_flashed = _signals.signal('message-flashed')

python/flask/templating.py

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.templating
+    ~~~~~~~~~~~~~~~~
+
+    Implements the bridge to Jinja2.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from jinja2 import BaseLoader, Environment as BaseEnvironment, \
+     TemplateNotFound
+
+from .globals import _request_ctx_stack, _app_ctx_stack
+from .signals import template_rendered, before_render_template
+
+
+def _default_template_ctx_processor():
+    """Default template context processor.  Injects `request`,
+    `session` and `g`.
+    """
+    reqctx = _request_ctx_stack.top
+    appctx = _app_ctx_stack.top
+    rv = {}
+    if appctx is not None:
+        rv['g'] = appctx.g
+    if reqctx is not None:
+        rv['request'] = reqctx.request
+        rv['session'] = reqctx.session
+    return rv
+
+
+class Environment(BaseEnvironment):
+    """Works like a regular Jinja2 environment but has some additional
+    knowledge of how Flask's blueprint works so that it can prepend the
+    name of the blueprint to referenced templates if necessary.
+    """
+
+    def __init__(self, app, **options):
+        if 'loader' not in options:
+            options['loader'] = app.create_global_jinja_loader()
+        BaseEnvironment.__init__(self, **options)
+        self.app = app
+
+
+class DispatchingJinjaLoader(BaseLoader):
+    """A loader that looks for templates in the application and all
+    the blueprint folders.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    def get_source(self, environment, template):
+        if self.app.config['EXPLAIN_TEMPLATE_LOADING']:
+            return self._get_source_explained(environment, template)
+        return self._get_source_fast(environment, template)
+
+    def _get_source_explained(self, environment, template):
+        attempts = []
+        trv = None
+
+        for srcobj, loader in self._iter_loaders(template):
+            try:
+                rv = loader.get_source(environment, template)
+                if trv is None:
+                    trv = rv
+            except TemplateNotFound:
+                rv = None
+            attempts.append((loader, srcobj, rv))
+
+        from .debughelpers import explain_template_loading_attempts
+        explain_template_loading_attempts(self.app, template, attempts)
+
+        if trv is not None:
+            return trv
+        raise TemplateNotFound(template)
+
+    def _get_source_fast(self, environment, template):
+        for srcobj, loader in self._iter_loaders(template):
+            try:
+                return loader.get_source(environment, template)
+            except TemplateNotFound:
+                continue
+        raise TemplateNotFound(template)
+
+    def _iter_loaders(self, template):
+        loader = self.app.jinja_loader
+        if loader is not None:
+            yield self.app, loader
+
+        for blueprint in self.app.iter_blueprints():
+            loader = blueprint.jinja_loader
+            if loader is not None:
+                yield blueprint, loader
+
+    def list_templates(self):
+        result = set()
+        loader = self.app.jinja_loader
+        if loader is not None:
+            result.update(loader.list_templates())
+
+        for blueprint in self.app.iter_blueprints():
+            loader = blueprint.jinja_loader
+            if loader is not None:
+                for template in loader.list_templates():
+                    result.add(template)
+
+        return list(result)
+
+
+def _render(template, context, app):
+    """Renders the template and fires the signal"""
+
+    before_render_template.send(app, template=template, context=context)
+    rv = template.render(context)
+    template_rendered.send(app, template=template, context=context)
+    return rv
+
+
+def render_template(template_name_or_list, **context):
+    """Renders a template from the template folder with the given
+    context.
+
+    :param template_name_or_list: the name of the template to be
+                                  rendered, or an iterable with template names
+                                  the first one existing will be rendered
+    :param context: the variables that should be available in the
+                    context of the template.
+    """
+    ctx = _app_ctx_stack.top
+    ctx.app.update_template_context(context)
+    return _render(ctx.app.jinja_env.get_or_select_template(template_name_or_list),
+                   context, ctx.app)
+
+
+def render_template_string(source, **context):
+    """Renders a template from the given template source string
+    with the given context. Template variables will be autoescaped.
+
+    :param source: the source code of the template to be
+                   rendered
+    :param context: the variables that should be available in the
+                    context of the template.
+    """
+    ctx = _app_ctx_stack.top
+    ctx.app.update_template_context(context)
+    return _render(ctx.app.jinja_env.from_string(source),
+                   context, ctx.app)

python/flask/testing.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.testing
+    ~~~~~~~~~~~~~
+
+    Implements test support helpers.  This module is lazily imported
+    and usually not used in production environments.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import werkzeug
+from contextlib import contextmanager
+
+from click.testing import CliRunner
+from flask.cli import ScriptInfo
+from werkzeug.test import Client, EnvironBuilder
+from flask import _request_ctx_stack
+from flask.json import dumps as json_dumps
+from werkzeug.urls import url_parse
+
+
+def make_test_environ_builder(
+    app, path='/', base_url=None, subdomain=None, url_scheme=None,
+    *args, **kwargs
+):
+    """Create a :class:`~werkzeug.test.EnvironBuilder`, taking some
+    defaults from the application.
+
+    :param app: The Flask application to configure the environment from.
+    :param path: URL path being requested.
+    :param base_url: Base URL where the app is being served, which
+        ``path`` is relative to. If not given, built from
+        :data:`PREFERRED_URL_SCHEME`, ``subdomain``,
+        :data:`SERVER_NAME`, and :data:`APPLICATION_ROOT`.
+    :param subdomain: Subdomain name to append to :data:`SERVER_NAME`.
+    :param url_scheme: Scheme to use instead of
+        :data:`PREFERRED_URL_SCHEME`.
+    :param json: If given, this is serialized as JSON and passed as
+        ``data``. Also defaults ``content_type`` to
+        ``application/json``.
+    :param args: other positional arguments passed to
+        :class:`~werkzeug.test.EnvironBuilder`.
+    :param kwargs: other keyword arguments passed to
+        :class:`~werkzeug.test.EnvironBuilder`.
+    """
+
+    assert (
+        not (base_url or subdomain or url_scheme)
+        or (base_url is not None) != bool(subdomain or url_scheme)
+    ), 'Cannot pass "subdomain" or "url_scheme" with "base_url".'
+
+    if base_url is None:
+        http_host = app.config.get('SERVER_NAME') or 'localhost'
+        app_root = app.config['APPLICATION_ROOT']
+
+        if subdomain:
+            http_host = '{0}.{1}'.format(subdomain, http_host)
+
+        if url_scheme is None:
+            url_scheme = app.config['PREFERRED_URL_SCHEME']
+
+        url = url_parse(path)
+        base_url = '{scheme}://{netloc}/{path}'.format(
+            scheme=url.scheme or url_scheme,
+            netloc=url.netloc or http_host,
+            path=app_root.lstrip('/')
+        )
+        path = url.path
+
+        if url.query:
+            sep = b'?' if isinstance(url.query, bytes) else '?'
+            path += sep + url.query
+
+    # TODO use EnvironBuilder.json_dumps once we require Werkzeug 0.15
+    if 'json' in kwargs:
+        assert 'data' not in kwargs, "Client cannot provide both 'json' and 'data'."
+        kwargs['data'] = json_dumps(kwargs.pop('json'), app=app)
+
+        if 'content_type' not in kwargs:
+            kwargs['content_type'] = 'application/json'
+
+    return EnvironBuilder(path, base_url, *args, **kwargs)
+
+
+class FlaskClient(Client):
+    """Works like a regular Werkzeug test client but has some knowledge about
+    how Flask works to defer the cleanup of the request context stack to the
+    end of a ``with`` body when used in a ``with`` statement.  For general
+    information about how to use this class refer to
+    :class:`werkzeug.test.Client`.
+
+    .. versionchanged:: 0.12
+       `app.test_client()` includes preset default environment, which can be
+       set after instantiation of the `app.test_client()` object in
+       `client.environ_base`.
+
+    Basic usage is outlined in the :ref:`testing` chapter.
+    """
+
+    preserve_context = False
+
+    def __init__(self, *args, **kwargs):
+        super(FlaskClient, self).__init__(*args, **kwargs)
+        self.environ_base = {
+            "REMOTE_ADDR": "127.0.0.1",
+            "HTTP_USER_AGENT": "werkzeug/" + werkzeug.__version__
+        }
+
+    @contextmanager
+    def session_transaction(self, *args, **kwargs):
+        """When used in combination with a ``with`` statement this opens a
+        session transaction.  This can be used to modify the session that
+        the test client uses.  Once the ``with`` block is left the session is
+        stored back.
+
+        ::
+
+            with client.session_transaction() as session:
+                session['value'] = 42
+
+        Internally this is implemented by going through a temporary test
+        request context and since session handling could depend on
+        request variables this function accepts the same arguments as
+        :meth:`~flask.Flask.test_request_context` which are directly
+        passed through.
+        """
+        if self.cookie_jar is None:
+            raise RuntimeError('Session transactions only make sense '
+                               'with cookies enabled.')
+        app = self.application
+        environ_overrides = kwargs.setdefault('environ_overrides', {})
+        self.cookie_jar.inject_wsgi(environ_overrides)
+        outer_reqctx = _request_ctx_stack.top
+        with app.test_request_context(*args, **kwargs) as c:
+            session_interface = app.session_interface
+            sess = session_interface.open_session(app, c.request)
+            if sess is None:
+                raise RuntimeError('Session backend did not open a session. '
+                                   'Check the configuration')
+
+            # Since we have to open a new request context for the session
+            # handling we want to make sure that we hide out own context
+            # from the caller.  By pushing the original request context
+            # (or None) on top of this and popping it we get exactly that
+            # behavior.  It's important to not use the push and pop
+            # methods of the actual request context object since that would
+            # mean that cleanup handlers are called
+            _request_ctx_stack.push(outer_reqctx)
+            try:
+                yield sess
+            finally:
+                _request_ctx_stack.pop()
+
+            resp = app.response_class()
+            if not session_interface.is_null_session(sess):
+                session_interface.save_session(app, sess, resp)
+            headers = resp.get_wsgi_headers(c.request.environ)
+            self.cookie_jar.extract_wsgi(c.request.environ, headers)
+
+    def open(self, *args, **kwargs):
+        as_tuple = kwargs.pop('as_tuple', False)
+        buffered = kwargs.pop('buffered', False)
+        follow_redirects = kwargs.pop('follow_redirects', False)
+
+        if (
+            not kwargs and len(args) == 1
+            and isinstance(args[0], (EnvironBuilder, dict))
+        ):
+            environ = self.environ_base.copy()
+
+            if isinstance(args[0], EnvironBuilder):
+                environ.update(args[0].get_environ())
+            else:
+                environ.update(args[0])
+
+            environ['flask._preserve_context'] = self.preserve_context
+        else:
+            kwargs.setdefault('environ_overrides', {}) \
+                ['flask._preserve_context'] = self.preserve_context
+            kwargs.setdefault('environ_base', self.environ_base)
+            builder = make_test_environ_builder(
+                self.application, *args, **kwargs
+            )
+
+            try:
+                environ = builder.get_environ()
+            finally:
+                builder.close()
+
+        return Client.open(
+            self, environ,
+            as_tuple=as_tuple,
+            buffered=buffered,
+            follow_redirects=follow_redirects
+        )
+
+    def __enter__(self):
+        if self.preserve_context:
+            raise RuntimeError('Cannot nest client invocations')
+        self.preserve_context = True
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.preserve_context = False
+
+        # on exit we want to clean up earlier.  Normally the request context
+        # stays preserved until the next request in the same thread comes
+        # in.  See RequestGlobals.push() for the general behavior.
+        top = _request_ctx_stack.top
+        if top is not None and top.preserved:
+            top.pop()
+
+
+class FlaskCliRunner(CliRunner):
+    """A :class:`~click.testing.CliRunner` for testing a Flask app's
+    CLI commands. Typically created using
+    :meth:`~flask.Flask.test_cli_runner`. See :ref:`testing-cli`.
+    """
+    def __init__(self, app, **kwargs):
+        self.app = app
+        super(FlaskCliRunner, self).__init__(**kwargs)
+
+    def invoke(self, cli=None, args=None, **kwargs):
+        """Invokes a CLI command in an isolated environment. See
+        :meth:`CliRunner.invoke <click.testing.CliRunner.invoke>` for
+        full method documentation. See :ref:`testing-cli` for examples.
+
+        If the ``obj`` argument is not given, passes an instance of
+        :class:`~flask.cli.ScriptInfo` that knows how to load the Flask
+        app being tested.
+
+        :param cli: Command object to invoke. Default is the app's
+            :attr:`~flask.app.Flask.cli` group.
+        :param args: List of strings to invoke the command with.
+
+        :return: a :class:`~click.testing.Result` object.
+        """
+        if cli is None:
+            cli = self.app.cli
+
+        if 'obj' not in kwargs:
+            kwargs['obj'] = ScriptInfo(create_app=lambda: self.app)
+
+        return super(FlaskCliRunner, self).invoke(cli, args, **kwargs)

python/flask/views.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.views
+    ~~~~~~~~~~~
+
+    This module provides class-based views inspired by the ones in Django.
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from .globals import request
+from ._compat import with_metaclass
+
+
+http_method_funcs = frozenset(['get', 'post', 'head', 'options',
+                               'delete', 'put', 'trace', 'patch'])
+
+
+class View(object):
+    """Alternative way to use view functions.  A subclass has to implement
+    :meth:`dispatch_request` which is called with the view arguments from
+    the URL routing system.  If :attr:`methods` is provided the methods
+    do not have to be passed to the :meth:`~flask.Flask.add_url_rule`
+    method explicitly::
+
+        class MyView(View):
+            methods = ['GET']
+
+            def dispatch_request(self, name):
+                return 'Hello %s!' % name
+
+        app.add_url_rule('/hello/<name>', view_func=MyView.as_view('myview'))
+
+    When you want to decorate a pluggable view you will have to either do that
+    when the view function is created (by wrapping the return value of
+    :meth:`as_view`) or you can use the :attr:`decorators` attribute::
+
+        class SecretView(View):
+            methods = ['GET']
+            decorators = [superuser_required]
+
+            def dispatch_request(self):
+                ...
+
+    The decorators stored in the decorators list are applied one after another
+    when the view function is created.  Note that you can *not* use the class
+    based decorators since those would decorate the view class and not the
+    generated view function!
+    """
+
+    #: A list of methods this view can handle.
+    methods = None
+
+    #: Setting this disables or force-enables the automatic options handling.
+    provide_automatic_options = None
+
+    #: The canonical way to decorate class-based views is to decorate the
+    #: return value of as_view().  However since this moves parts of the
+    #: logic from the class declaration to the place where it's hooked
+    #: into the routing system.
+    #:
+    #: You can place one or more decorators in this list and whenever the
+    #: view function is created the result is automatically decorated.
+    #:
+    #: .. versionadded:: 0.8
+    decorators = ()
+
+    def dispatch_request(self):
+        """Subclasses have to override this method to implement the
+        actual view function code.  This method is called with all
+        the arguments from the URL rule.
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def as_view(cls, name, *class_args, **class_kwargs):
+        """Converts the class into an actual view function that can be used
+        with the routing system.  Internally this generates a function on the
+        fly which will instantiate the :class:`View` on each request and call
+        the :meth:`dispatch_request` method on it.
+
+        The arguments passed to :meth:`as_view` are forwarded to the
+        constructor of the class.
+        """
+        def view(*args, **kwargs):
+            self = view.view_class(*class_args, **class_kwargs)
+            return self.dispatch_request(*args, **kwargs)
+
+        if cls.decorators:
+            view.__name__ = name
+            view.__module__ = cls.__module__
+            for decorator in cls.decorators:
+                view = decorator(view)
+
+        # We attach the view class to the view function for two reasons:
+        # first of all it allows us to easily figure out what class-based
+        # view this thing came from, secondly it's also used for instantiating
+        # the view class so you can actually replace it with something else
+        # for testing purposes and debugging.
+        view.view_class = cls
+        view.__name__ = name
+        view.__doc__ = cls.__doc__
+        view.__module__ = cls.__module__
+        view.methods = cls.methods
+        view.provide_automatic_options = cls.provide_automatic_options
+        return view
+
+
+class MethodViewType(type):
+    """Metaclass for :class:`MethodView` that determines what methods the view
+    defines.
+    """
+
+    def __init__(cls, name, bases, d):
+        super(MethodViewType, cls).__init__(name, bases, d)
+
+        if 'methods' not in d:
+            methods = set()
+
+            for key in http_method_funcs:
+                if hasattr(cls, key):
+                    methods.add(key.upper())
+
+            # If we have no method at all in there we don't want to add a
+            # method list. This is for instance the case for the base class
+            # or another subclass of a base method view that does not introduce
+            # new methods.
+            if methods:
+                cls.methods = methods
+
+
+class MethodView(with_metaclass(MethodViewType, View)):
+    """A class-based view that dispatches request methods to the corresponding
+    class methods. For example, if you implement a ``get`` method, it will be
+    used to handle ``GET`` requests. ::
+
+        class CounterAPI(MethodView):
+            def get(self):
+                return session.get('counter', 0)
+
+            def post(self):
+                session['counter'] = session.get('counter', 0) + 1
+                return 'OK'
+
+        app.add_url_rule('/counter', view_func=CounterAPI.as_view('counter'))
+    """
+
+    def dispatch_request(self, *args, **kwargs):
+        meth = getattr(self, request.method.lower(), None)
+
+        # If the request method is HEAD and we don't have a handler for it
+        # retry with GET.
+        if meth is None and request.method == 'HEAD':
+            meth = getattr(self, 'get', None)
+
+        assert meth is not None, 'Unimplemented method %r' % request.method
+        return meth(*args, **kwargs)

python/flask/wrappers.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.wrappers
+    ~~~~~~~~~~~~~~
+
+    Implements the WSGI wrappers (request and response).
+
+    :copyright: © 2010 by the Pallets team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from werkzeug.exceptions import BadRequest
+from werkzeug.wrappers import Request as RequestBase, Response as ResponseBase
+
+from flask import json
+from flask.globals import current_app
+
+
+class JSONMixin(object):
+    """Common mixin for both request and response objects to provide JSON
+    parsing capabilities.
+
+    .. versionadded:: 1.0
+    """
+
+    _cached_json = (Ellipsis, Ellipsis)
+
+    @property
+    def is_json(self):
+        """Check if the mimetype indicates JSON data, either
+        :mimetype:`application/json` or :mimetype:`application/*+json`.
+
+        .. versionadded:: 0.11
+        """
+        mt = self.mimetype
+        return (
+            mt == 'application/json'
+            or (mt.startswith('application/')) and mt.endswith('+json')
+        )
+
+    @property
+    def json(self):
+        """This will contain the parsed JSON data if the mimetype indicates
+        JSON (:mimetype:`application/json`, see :meth:`is_json`), otherwise it
+        will be ``None``.
+        """
+        return self.get_json()
+
+    def _get_data_for_json(self, cache):
+        return self.get_data(cache=cache)
+
+    def get_json(self, force=False, silent=False, cache=True):
+        """Parse and return the data as JSON. If the mimetype does not
+        indicate JSON (:mimetype:`application/json`, see
+        :meth:`is_json`), this returns ``None`` unless ``force`` is
+        true. If parsing fails, :meth:`on_json_loading_failed` is called
+        and its return value is used as the return value.
+
+        :param force: Ignore the mimetype and always try to parse JSON.
+        :param silent: Silence parsing errors and return ``None``
+            instead.
+        :param cache: Store the parsed JSON to return for subsequent
+            calls.
+        """
+        if cache and self._cached_json[silent] is not Ellipsis:
+            return self._cached_json[silent]
+
+        if not (force or self.is_json):
+            return None
+
+        data = self._get_data_for_json(cache=cache)
+
+        try:
+            rv = json.loads(data)
+        except ValueError as e:
+            if silent:
+                rv = None
+                if cache:
+                    normal_rv, _ = self._cached_json
+                    self._cached_json = (normal_rv, rv)
+            else:
+                rv = self.on_json_loading_failed(e)
+                if cache:
+                    _, silent_rv = self._cached_json
+                    self._cached_json = (rv, silent_rv)
+        else:
+            if cache:
+                self._cached_json = (rv, rv)
+
+        return rv
+
+    def on_json_loading_failed(self, e):
+        """Called if :meth:`get_json` parsing fails and isn't silenced. If
+        this method returns a value, it is used as the return value for
+        :meth:`get_json`. The default implementation raises a
+        :class:`BadRequest` exception.
+
+        .. versionchanged:: 0.10
+           Raise a :exc:`BadRequest` error instead of returning an error
+           message as JSON. If you want that behavior you can add it by
+           subclassing.
+
+        .. versionadded:: 0.8
+        """
+        if current_app is not None and current_app.debug:
+            raise BadRequest('Failed to decode JSON object: {0}'.format(e))
+
+        raise BadRequest()
+
+
+class Request(RequestBase, JSONMixin):
+    """The request object used by default in Flask.  Remembers the
+    matched endpoint and view arguments.
+
+    It is what ends up as :class:`~flask.request`.  If you want to replace
+    the request object used you can subclass this and set
+    :attr:`~flask.Flask.request_class` to your subclass.
+
+    The request object is a :class:`~werkzeug.wrappers.Request` subclass and
+    provides all of the attributes Werkzeug defines plus a few Flask
+    specific ones.
+    """
+
+    #: The internal URL rule that matched the request.  This can be
+    #: useful to inspect which methods are allowed for the URL from
+    #: a before/after handler (``request.url_rule.methods``) etc.
+    #: Though if the request's method was invalid for the URL rule,
+    #: the valid list is available in ``routing_exception.valid_methods``
+    #: instead (an attribute of the Werkzeug exception :exc:`~werkzeug.exceptions.MethodNotAllowed`)
+    #: because the request was never internally bound.
+    #:
+    #: .. versionadded:: 0.6
+    url_rule = None
+
+    #: A dict of view arguments that matched the request.  If an exception
+    #: happened when matching, this will be ``None``.
+    view_args = None
+
+    #: If matching the URL failed, this is the exception that will be
+    #: raised / was raised as part of the request handling.  This is
+    #: usually a :exc:`~werkzeug.exceptions.NotFound` exception or
+    #: something similar.
+    routing_exception = None
+
+    @property
+    def max_content_length(self):
+        """Read-only view of the ``MAX_CONTENT_LENGTH`` config key."""
+        if current_app:
+            return current_app.config['MAX_CONTENT_LENGTH']
+
+    @property
+    def endpoint(self):
+        """The endpoint that matched the request.  This in combination with
+        :attr:`view_args` can be used to reconstruct the same or a
+        modified URL.  If an exception happened when matching, this will
+        be ``None``.
+        """
+        if self.url_rule is not None:
+            return self.url_rule.endpoint
+
+    @property
+    def blueprint(self):
+        """The name of the current blueprint"""
+        if self.url_rule and '.' in self.url_rule.endpoint:
+            return self.url_rule.endpoint.rsplit('.', 1)[0]
+
+    def _load_form_data(self):
+        RequestBase._load_form_data(self)
+
+        # In debug mode we're replacing the files multidict with an ad-hoc
+        # subclass that raises a different error for key errors.
+        if (
+            current_app
+            and current_app.debug
+            and self.mimetype != 'multipart/form-data'
+            and not self.files
+        ):
+            from .debughelpers import attach_enctype_error_multidict
+            attach_enctype_error_multidict(self)
+
+
+class Response(ResponseBase, JSONMixin):
+    """The response object that is used by default in Flask.  Works like the
+    response object from Werkzeug but is set to have an HTML mimetype by
+    default.  Quite often you don't have to create this object yourself because
+    :meth:`~flask.Flask.make_response` will take care of that for you.
+
+    If you want to replace the response object used you can subclass this and
+    set :attr:`~flask.Flask.response_class` to your subclass.
+
+    .. versionchanged:: 1.0
+        JSON support is added to the response, like the request. This is useful
+        when testing to get the test client response data as JSON.
+
+    .. versionchanged:: 1.0
+
+        Added :attr:`max_cookie_size`.
+    """
+
+    default_mimetype = 'text/html'
+
+    def _get_data_for_json(self, cache):
+        return self.get_data()
+
+    @property
+    def max_cookie_size(self):
+        """Read-only view of the :data:`MAX_COOKIE_SIZE` config key.
+
+        See :attr:`~werkzeug.wrappers.BaseResponse.max_cookie_size` in
+        Werkzeug's docs.
+        """
+        if current_app:
+            return current_app.config['MAX_COOKIE_SIZE']
+
+        # return Werkzeug's default when not in an app context
+        return super(Response, self).max_cookie_size

python/itsdangerous/__init__.py

@@ -0,0 +1,22 @@
+from ._json import json
+from .encoding import base64_decode
+from .encoding import base64_encode
+from .encoding import want_bytes
+from .exc import BadData
+from .exc import BadHeader
+from .exc import BadPayload
+from .exc import BadSignature
+from .exc import BadTimeSignature
+from .exc import SignatureExpired
+from .jws import JSONWebSignatureSerializer
+from .jws import TimedJSONWebSignatureSerializer
+from .serializer import Serializer
+from .signer import HMACAlgorithm
+from .signer import NoneAlgorithm
+from .signer import Signer
+from .timed import TimedSerializer
+from .timed import TimestampSigner
+from .url_safe import URLSafeSerializer
+from .url_safe import URLSafeTimedSerializer
+
+__version__ = "1.1.0"

python/itsdangerous/_compat.py

@@ -0,0 +1,46 @@
+import decimal
+import hmac
+import numbers
+import sys
+
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from itertools import izip
+
+    text_type = unicode  # noqa: 821
+else:
+    izip = zip
+    text_type = str
+
+number_types = (numbers.Real, decimal.Decimal)
+
+
+def _constant_time_compare(val1, val2):
+    """Return ``True`` if the two strings are equal, ``False``
+    otherwise.
+
+    The time taken is independent of the number of characters that
+    match. Do not use this function for anything else than comparision
+    with known length targets.
+
+    This is should be implemented in C in order to get it completely
+    right.
+
+    This is an alias of :func:`hmac.compare_digest` on Python>=2.7,3.3.
+    """
+    len_eq = len(val1) == len(val2)
+    if len_eq:
+        result = 0
+        left = val1
+    else:
+        result = 1
+        left = val2
+    for x, y in izip(bytearray(left), bytearray(val2)):
+        result |= x ^ y
+    return result == 0
+
+
+# Starting with 2.7/3.3 the standard library has a c-implementation for
+# constant time string compares.
+constant_time_compare = getattr(hmac, "compare_digest", _constant_time_compare)

python/itsdangerous/_json.py

@@ -0,0 +1,18 @@
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+
+class _CompactJSON(object):
+    """Wrapper around json module that strips whitespace."""
+
+    @staticmethod
+    def loads(payload):
+        return json.loads(payload)
+
+    @staticmethod
+    def dumps(obj, **kwargs):
+        kwargs.setdefault("ensure_ascii", False)
+        kwargs.setdefault("separators", (",", ":"))
+        return json.dumps(obj, **kwargs)

python/itsdangerous/encoding.py

@@ -0,0 +1,49 @@
+import base64
+import string
+import struct
+
+from ._compat import text_type
+from .exc import BadData
+
+
+def want_bytes(s, encoding="utf-8", errors="strict"):
+    if isinstance(s, text_type):
+        s = s.encode(encoding, errors)
+    return s
+
+
+def base64_encode(string):
+    """Base64 encode a string of bytes or text. The resulting bytes are
+    safe to use in URLs.
+    """
+    string = want_bytes(string)
+    return base64.urlsafe_b64encode(string).rstrip(b"=")
+
+
+def base64_decode(string):
+    """Base64 decode a URL-safe string of bytes or text. The result is
+    bytes.
+    """
+    string = want_bytes(string, encoding="ascii", errors="ignore")
+    string += b"=" * (-len(string) % 4)
+
+    try:
+        return base64.urlsafe_b64decode(string)
+    except (TypeError, ValueError):
+        raise BadData("Invalid base64-encoded data")
+
+
+# The alphabet used by base64.urlsafe_*
+_base64_alphabet = (string.ascii_letters + string.digits + "-_=").encode("ascii")
+
+_int64_struct = struct.Struct(">Q")
+_int_to_bytes = _int64_struct.pack
+_bytes_to_int = _int64_struct.unpack
+
+
+def int_to_bytes(num):
+    return _int_to_bytes(num).lstrip(b"\x00")
+
+
+def bytes_to_int(bytestr):
+    return _bytes_to_int(bytestr.rjust(8, b"\x00"))[0]

python/itsdangerous/exc.py

@@ -0,0 +1,98 @@
+from ._compat import PY2
+from ._compat import text_type
+
+
+class BadData(Exception):
+    """Raised if bad data of any sort was encountered. This is the base
+    for all exceptions that itsdangerous defines.
+
+    .. versionadded:: 0.15
+    """
+
+    message = None
+
+    def __init__(self, message):
+        super(BadData, self).__init__(self, message)
+        self.message = message
+
+    def __str__(self):
+        return text_type(self.message)
+
+    if PY2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.__unicode__().encode("utf-8")
+
+
+class BadSignature(BadData):
+    """Raised if a signature does not match."""
+
+    def __init__(self, message, payload=None):
+        BadData.__init__(self, message)
+
+        #: The payload that failed the signature test. In some
+        #: situations you might still want to inspect this, even if
+        #: you know it was tampered with.
+        #:
+        #: .. versionadded:: 0.14
+        self.payload = payload
+
+
+class BadTimeSignature(BadSignature):
+    """Raised if a time-based signature is invalid. This is a subclass
+    of :class:`BadSignature`.
+    """
+
+    def __init__(self, message, payload=None, date_signed=None):
+        BadSignature.__init__(self, message, payload)
+
+        #: If the signature expired this exposes the date of when the
+        #: signature was created. This can be helpful in order to
+        #: tell the user how long a link has been gone stale.
+        #:
+        #: .. versionadded:: 0.14
+        self.date_signed = date_signed
+
+
+class SignatureExpired(BadTimeSignature):
+    """Raised if a signature timestamp is older than ``max_age``. This
+    is a subclass of :exc:`BadTimeSignature`.
+    """
+
+
+class BadHeader(BadSignature):
+    """Raised if a signed header is invalid in some form. This only
+    happens for serializers that have a header that goes with the
+    signature.
+
+    .. versionadded:: 0.24
+    """
+
+    def __init__(self, message, payload=None, header=None, original_error=None):
+        BadSignature.__init__(self, message, payload)
+
+        #: If the header is actually available but just malformed it
+        #: might be stored here.
+        self.header = header
+
+        #: If available, the error that indicates why the payload was
+        #: not valid. This might be ``None``.
+        self.original_error = original_error
+
+
+class BadPayload(BadData):
+    """Raised if a payload is invalid. This could happen if the payload
+    is loaded despite an invalid signature, or if there is a mismatch
+    between the serializer and deserializer. The original exception
+    that occurred during loading is stored on as :attr:`original_error`.
+
+    .. versionadded:: 0.15
+    """
+
+    def __init__(self, message, original_error=None):
+        BadData.__init__(self, message)
+
+        #: If available, the error that indicates why the payload was
+        #: not valid. This might be ``None``.
+        self.original_error = original_error

python/itsdangerous/jws.py

@@ -0,0 +1,218 @@
+import hashlib
+import time
+from datetime import datetime
+
+from ._compat import number_types
+from ._json import _CompactJSON
+from ._json import json
+from .encoding import base64_decode
+from .encoding import base64_encode
+from .encoding import want_bytes
+from .exc import BadData
+from .exc import BadHeader
+from .exc import BadPayload
+from .exc import BadSignature
+from .exc import SignatureExpired
+from .serializer import Serializer
+from .signer import HMACAlgorithm
+from .signer import NoneAlgorithm
+
+
+class JSONWebSignatureSerializer(Serializer):
+    """This serializer implements JSON Web Signature (JWS) support. Only
+    supports the JWS Compact Serialization.
+    """
+
+    jws_algorithms = {
+        "HS256": HMACAlgorithm(hashlib.sha256),
+        "HS384": HMACAlgorithm(hashlib.sha384),
+        "HS512": HMACAlgorithm(hashlib.sha512),
+        "none": NoneAlgorithm(),
+    }
+
+    #: The default algorithm to use for signature generation
+    default_algorithm = "HS512"
+
+    default_serializer = _CompactJSON
+
+    def __init__(
+        self,
+        secret_key,
+        salt=None,
+        serializer=None,
+        serializer_kwargs=None,
+        signer=None,
+        signer_kwargs=None,
+        algorithm_name=None,
+    ):
+        Serializer.__init__(
+            self,
+            secret_key=secret_key,
+            salt=salt,
+            serializer=serializer,
+            serializer_kwargs=serializer_kwargs,
+            signer=signer,
+            signer_kwargs=signer_kwargs,
+        )
+        if algorithm_name is None:
+            algorithm_name = self.default_algorithm
+        self.algorithm_name = algorithm_name
+        self.algorithm = self.make_algorithm(algorithm_name)
+
+    def load_payload(self, payload, serializer=None, return_header=False):
+        payload = want_bytes(payload)
+        if b"." not in payload:
+            raise BadPayload('No "." found in value')
+        base64d_header, base64d_payload = payload.split(b".", 1)
+        try:
+            json_header = base64_decode(base64d_header)
+        except Exception as e:
+            raise BadHeader(
+                "Could not base64 decode the header because of an exception",
+                original_error=e,
+            )
+        try:
+            json_payload = base64_decode(base64d_payload)
+        except Exception as e:
+            raise BadPayload(
+                "Could not base64 decode the payload because of an exception",
+                original_error=e,
+            )
+        try:
+            header = Serializer.load_payload(self, json_header, serializer=json)
+        except BadData as e:
+            raise BadHeader(
+                "Could not unserialize header because it was malformed",
+                original_error=e,
+            )
+        if not isinstance(header, dict):
+            raise BadHeader("Header payload is not a JSON object", header=header)
+        payload = Serializer.load_payload(self, json_payload, serializer=serializer)
+        if return_header:
+            return payload, header
+        return payload
+
+    def dump_payload(self, header, obj):
+        base64d_header = base64_encode(
+            self.serializer.dumps(header, **self.serializer_kwargs)
+        )
+        base64d_payload = base64_encode(
+            self.serializer.dumps(obj, **self.serializer_kwargs)
+        )
+        return base64d_header + b"." + base64d_payload
+
+    def make_algorithm(self, algorithm_name):
+        try:
+            return self.jws_algorithms[algorithm_name]
+        except KeyError:
+            raise NotImplementedError("Algorithm not supported")
+
+    def make_signer(self, salt=None, algorithm=None):
+        if salt is None:
+            salt = self.salt
+        key_derivation = "none" if salt is None else None
+        if algorithm is None:
+            algorithm = self.algorithm
+        return self.signer(
+            self.secret_key,
+            salt=salt,
+            sep=".",
+            key_derivation=key_derivation,
+            algorithm=algorithm,
+        )
+
+    def make_header(self, header_fields):
+        header = header_fields.copy() if header_fields else {}
+        header["alg"] = self.algorithm_name
+        return header
+
+    def dumps(self, obj, salt=None, header_fields=None):
+        """Like :meth:`.Serializer.dumps` but creates a JSON Web
+        Signature. It also allows for specifying additional fields to be
+        included in the JWS header.
+        """
+        header = self.make_header(header_fields)
+        signer = self.make_signer(salt, self.algorithm)
+        return signer.sign(self.dump_payload(header, obj))
+
+    def loads(self, s, salt=None, return_header=False):
+        """Reverse of :meth:`dumps`. If requested via ``return_header``
+        it will return a tuple of payload and header.
+        """
+        payload, header = self.load_payload(
+            self.make_signer(salt, self.algorithm).unsign(want_bytes(s)),
+            return_header=True,
+        )
+        if header.get("alg") != self.algorithm_name:
+            raise BadHeader("Algorithm mismatch", header=header, payload=payload)
+        if return_header:
+            return payload, header
+        return payload
+
+    def loads_unsafe(self, s, salt=None, return_header=False):
+        kwargs = {"return_header": return_header}
+        return self._loads_unsafe_impl(s, salt, kwargs, kwargs)
+
+
+class TimedJSONWebSignatureSerializer(JSONWebSignatureSerializer):
+    """Works like the regular :class:`JSONWebSignatureSerializer` but
+    also records the time of the signing and can be used to expire
+    signatures.
+
+    JWS currently does not specify this behavior but it mentions a
+    possible extension like this in the spec. Expiry date is encoded
+    into the header similar to what's specified in `draft-ietf-oauth
+    -json-web-token <http://self-issued.info/docs/draft-ietf-oauth-json
+    -web-token.html#expDef>`_.
+    """
+
+    DEFAULT_EXPIRES_IN = 3600
+
+    def __init__(self, secret_key, expires_in=None, **kwargs):
+        JSONWebSignatureSerializer.__init__(self, secret_key, **kwargs)
+        if expires_in is None:
+            expires_in = self.DEFAULT_EXPIRES_IN
+        self.expires_in = expires_in
+
+    def make_header(self, header_fields):
+        header = JSONWebSignatureSerializer.make_header(self, header_fields)
+        iat = self.now()
+        exp = iat + self.expires_in
+        header["iat"] = iat
+        header["exp"] = exp
+        return header
+
+    def loads(self, s, salt=None, return_header=False):
+        payload, header = JSONWebSignatureSerializer.loads(
+            self, s, salt, return_header=True
+        )
+
+        if "exp" not in header:
+            raise BadSignature("Missing expiry date", payload=payload)
+
+        int_date_error = BadHeader("Expiry date is not an IntDate", payload=payload)
+        try:
+            header["exp"] = int(header["exp"])
+        except ValueError:
+            raise int_date_error
+        if header["exp"] < 0:
+            raise int_date_error
+
+        if header["exp"] < self.now():
+            raise SignatureExpired(
+                "Signature expired",
+                payload=payload,
+                date_signed=self.get_issue_date(header),
+            )
+
+        if return_header:
+            return payload, header
+        return payload
+
+    def get_issue_date(self, header):
+        rv = header.get("iat")
+        if isinstance(rv, number_types):
+            return datetime.utcfromtimestamp(int(rv))
+
+    def now(self):
+        return int(time.time())

python/itsdangerous/serializer.py

@@ -0,0 +1,233 @@
+import hashlib
+
+from ._compat import text_type
+from ._json import json
+from .encoding import want_bytes
+from .exc import BadPayload
+from .exc import BadSignature
+from .signer import Signer
+
+
+def is_text_serializer(serializer):
+    """Checks whether a serializer generates text or binary."""
+    return isinstance(serializer.dumps({}), text_type)
+
+
+class Serializer(object):
+    """This class provides a serialization interface on top of the
+    signer. It provides a similar API to json/pickle and other modules
+    but is structured differently internally. If you want to change the
+    underlying implementation for parsing and loading you have to
+    override the :meth:`load_payload` and :meth:`dump_payload`
+    functions.
+
+    This implementation uses simplejson if available for dumping and
+    loading and will fall back to the standard library's json module if
+    it's not available.
+
+    You do not need to subclass this class in order to switch out or
+    customize the :class:`.Signer`. You can instead pass a different
+    class to the constructor as well as keyword arguments as a dict that
+    should be forwarded.
+
+    .. code-block:: python
+
+        s = Serializer(signer_kwargs={'key_derivation': 'hmac'})
+
+    You may want to upgrade the signing parameters without invalidating
+    existing signatures that are in use. Fallback signatures can be
+    given that will be tried if unsigning with the current signer fails.
+
+    Fallback signers can be defined by providing a list of
+    ``fallback_signers``. Each item can be one of the following: a
+    signer class (which is instantiated with ``signer_kwargs``,
+    ``salt``, and ``secret_key``), a tuple
+    ``(signer_class, signer_kwargs)``, or a dict of ``signer_kwargs``.
+
+    For example, this is a serializer that signs using SHA-512, but will
+    unsign using either SHA-512 or SHA1:
+
+    .. code-block:: python
+
+        s = Serializer(
+            signer_kwargs={"digest_method": hashlib.sha512},
+            fallback_signers=[{"digest_method": hashlib.sha1}]
+        )
+
+    .. versionchanged:: 0.14:
+        The ``signer`` and ``signer_kwargs`` parameters were added to
+        the constructor.
+
+    .. versionchanged:: 1.1.0:
+        Added support for ``fallback_signers`` and configured a default
+        SHA-512 fallback. This fallback is for users who used the yanked
+        1.0.0 release which defaulted to SHA-512.
+    """
+
+    #: If a serializer module or class is not passed to the constructor
+    #: this one is picked up. This currently defaults to :mod:`json`.
+    default_serializer = json
+
+    #: The default :class:`Signer` class that is being used by this
+    #: serializer.
+    #:
+    #: .. versionadded:: 0.14
+    default_signer = Signer
+
+    #: The default fallback signers.
+    default_fallback_signers = [{"digest_method": hashlib.sha512}]
+
+    def __init__(
+        self,
+        secret_key,
+        salt=b"itsdangerous",
+        serializer=None,
+        serializer_kwargs=None,
+        signer=None,
+        signer_kwargs=None,
+        fallback_signers=None,
+    ):
+        self.secret_key = want_bytes(secret_key)
+        self.salt = want_bytes(salt)
+        if serializer is None:
+            serializer = self.default_serializer
+        self.serializer = serializer
+        self.is_text_serializer = is_text_serializer(serializer)
+        if signer is None:
+            signer = self.default_signer
+        self.signer = signer
+        self.signer_kwargs = signer_kwargs or {}
+        if fallback_signers is None:
+            fallback_signers = list(self.default_fallback_signers or ())
+        self.fallback_signers = fallback_signers
+        self.serializer_kwargs = serializer_kwargs or {}
+
+    def load_payload(self, payload, serializer=None):
+        """Loads the encoded object. This function raises
+        :class:`.BadPayload` if the payload is not valid. The
+        ``serializer`` parameter can be used to override the serializer
+        stored on the class. The encoded ``payload`` should always be
+        bytes.
+        """
+        if serializer is None:
+            serializer = self.serializer
+            is_text = self.is_text_serializer
+        else:
+            is_text = is_text_serializer(serializer)
+        try:
+            if is_text:
+                payload = payload.decode("utf-8")
+            return serializer.loads(payload)
+        except Exception as e:
+            raise BadPayload(
+                "Could not load the payload because an exception"
+                " occurred on unserializing the data.",
+                original_error=e,
+            )
+
+    def dump_payload(self, obj):
+        """Dumps the encoded object. The return value is always bytes.
+        If the internal serializer returns text, the value will be
+        encoded as UTF-8.
+        """
+        return want_bytes(self.serializer.dumps(obj, **self.serializer_kwargs))
+
+    def make_signer(self, salt=None):
+        """Creates a new instance of the signer to be used. The default
+        implementation uses the :class:`.Signer` base class.
+        """
+        if salt is None:
+            salt = self.salt
+        return self.signer(self.secret_key, salt=salt, **self.signer_kwargs)
+
+    def iter_unsigners(self, salt=None):
+        """Iterates over all signers to be tried for unsigning. Starts
+        with the configured signer, then constructs each signer
+        specified in ``fallback_signers``.
+        """
+        if salt is None:
+            salt = self.salt
+        yield self.make_signer(salt)
+        for fallback in self.fallback_signers:
+            if type(fallback) is dict:
+                kwargs = fallback
+                fallback = self.signer
+            elif type(fallback) is tuple:
+                fallback, kwargs = fallback
+            else:
+                kwargs = self.signer_kwargs
+            yield fallback(self.secret_key, salt=salt, **kwargs)
+
+    def dumps(self, obj, salt=None):
+        """Returns a signed string serialized with the internal
+        serializer. The return value can be either a byte or unicode
+        string depending on the format of the internal serializer.
+        """
+        payload = want_bytes(self.dump_payload(obj))
+        rv = self.make_signer(salt).sign(payload)
+        if self.is_text_serializer:
+            rv = rv.decode("utf-8")
+        return rv
+
+    def dump(self, obj, f, salt=None):
+        """Like :meth:`dumps` but dumps into a file. The file handle has
+        to be compatible with what the internal serializer expects.
+        """
+        f.write(self.dumps(obj, salt))
+
+    def loads(self, s, salt=None):
+        """Reverse of :meth:`dumps`. Raises :exc:`.BadSignature` if the
+        signature validation fails.
+        """
+        s = want_bytes(s)
+        last_exception = None
+        for signer in self.iter_unsigners(salt):
+            try:
+                return self.load_payload(signer.unsign(s))
+            except BadSignature as err:
+                last_exception = err
+        raise last_exception
+
+    def load(self, f, salt=None):
+        """Like :meth:`loads` but loads from a file."""
+        return self.loads(f.read(), salt)
+
+    def loads_unsafe(self, s, salt=None):
+        """Like :meth:`loads` but without verifying the signature. This
+        is potentially very dangerous to use depending on how your
+        serializer works. The return value is ``(signature_valid,
+        payload)`` instead of just the payload. The first item will be a
+        boolean that indicates if the signature is valid. This function
+        never fails.
+
+        Use it for debugging only and if you know that your serializer
+        module is not exploitable (for example, do not use it with a
+        pickle serializer).
+
+        .. versionadded:: 0.15
+        """
+        return self._loads_unsafe_impl(s, salt)
+
+    def _loads_unsafe_impl(self, s, salt, load_kwargs=None, load_payload_kwargs=None):
+        """Low level helper function to implement :meth:`loads_unsafe`
+        in serializer subclasses.
+        """
+        try:
+            return True, self.loads(s, salt=salt, **(load_kwargs or {}))
+        except BadSignature as e:
+            if e.payload is None:
+                return False, None
+            try:
+                return (
+                    False,
+                    self.load_payload(e.payload, **(load_payload_kwargs or {})),
+                )
+            except BadPayload:
+                return False, None
+
+    def load_unsafe(self, f, *args, **kwargs):
+        """Like :meth:`loads_unsafe` but loads from a file.
+
+        .. versionadded:: 0.15
+        """
+        return self.loads_unsafe(f.read(), *args, **kwargs)

python/itsdangerous/signer.py

@@ -0,0 +1,179 @@
+import hashlib
+import hmac
+
+from ._compat import constant_time_compare
+from .encoding import _base64_alphabet
+from .encoding import base64_decode
+from .encoding import base64_encode
+from .encoding import want_bytes
+from .exc import BadSignature
+
+
+class SigningAlgorithm(object):
+    """Subclasses must implement :meth:`get_signature` to provide
+    signature generation functionality.
+    """
+
+    def get_signature(self, key, value):
+        """Returns the signature for the given key and value."""
+        raise NotImplementedError()
+
+    def verify_signature(self, key, value, sig):
+        """Verifies the given signature matches the expected
+        signature.
+        """
+        return constant_time_compare(sig, self.get_signature(key, value))
+
+
+class NoneAlgorithm(SigningAlgorithm):
+    """Provides an algorithm that does not perform any signing and
+    returns an empty signature.
+    """
+
+    def get_signature(self, key, value):
+        return b""
+
+
+class HMACAlgorithm(SigningAlgorithm):
+    """Provides signature generation using HMACs."""
+
+    #: The digest method to use with the MAC algorithm. This defaults to
+    #: SHA1, but can be changed to any other function in the hashlib
+    #: module.
+    default_digest_method = staticmethod(hashlib.sha1)
+
+    def __init__(self, digest_method=None):
+        if digest_method is None:
+            digest_method = self.default_digest_method
+        self.digest_method = digest_method
+
+    def get_signature(self, key, value):
+        mac = hmac.new(key, msg=value, digestmod=self.digest_method)
+        return mac.digest()
+
+
+class Signer(object):
+    """This class can sign and unsign bytes, validating the signature
+    provided.
+
+    Salt can be used to namespace the hash, so that a signed string is
+    only valid for a given namespace. Leaving this at the default value
+    or re-using a salt value across different parts of your application
+    where the same signed value in one part can mean something different
+    in another part is a security risk.
+
+    See :ref:`the-salt` for an example of what the salt is doing and how
+    you can utilize it.
+
+    .. versionadded:: 0.14
+        ``key_derivation`` and ``digest_method`` were added as arguments
+        to the class constructor.
+
+    .. versionadded:: 0.18
+        ``algorithm`` was added as an argument to the class constructor.
+    """
+
+    #: The digest method to use for the signer.  This defaults to
+    #: SHA1 but can be changed to any other function in the hashlib
+    #: module.
+    #:
+    #: .. versionadded:: 0.14
+    default_digest_method = staticmethod(hashlib.sha1)
+
+    #: Controls how the key is derived. The default is Django-style
+    #: concatenation. Possible values are ``concat``, ``django-concat``
+    #: and ``hmac``. This is used for deriving a key from the secret key
+    #: with an added salt.
+    #:
+    #: .. versionadded:: 0.14
+    default_key_derivation = "django-concat"
+
+    def __init__(
+        self,
+        secret_key,
+        salt=None,
+        sep=".",
+        key_derivation=None,
+        digest_method=None,
+        algorithm=None,
+    ):
+        self.secret_key = want_bytes(secret_key)
+        self.sep = want_bytes(sep)
+        if self.sep in _base64_alphabet:
+            raise ValueError(
+                "The given separator cannot be used because it may be"
+                " contained in the signature itself. Alphanumeric"
+                " characters and `-_=` must not be used."
+            )
+        self.salt = "itsdangerous.Signer" if salt is None else salt
+        if key_derivation is None:
+            key_derivation = self.default_key_derivation
+        self.key_derivation = key_derivation
+        if digest_method is None:
+            digest_method = self.default_digest_method
+        self.digest_method = digest_method
+        if algorithm is None:
+            algorithm = HMACAlgorithm(self.digest_method)
+        self.algorithm = algorithm
+
+    def derive_key(self):
+        """This method is called to derive the key. The default key
+        derivation choices can be overridden here. Key derivation is not
+        intended to be used as a security method to make a complex key
+        out of a short password. Instead you should use large random
+        secret keys.
+        """
+        salt = want_bytes(self.salt)
+        if self.key_derivation == "concat":
+            return self.digest_method(salt + self.secret_key).digest()
+        elif self.key_derivation == "django-concat":
+            return self.digest_method(salt + b"signer" + self.secret_key).digest()
+        elif self.key_derivation == "hmac":
+            mac = hmac.new(self.secret_key, digestmod=self.digest_method)
+            mac.update(salt)
+            return mac.digest()
+        elif self.key_derivation == "none":
+            return self.secret_key
+        else:
+            raise TypeError("Unknown key derivation method")
+
+    def get_signature(self, value):
+        """Returns the signature for the given value."""
+        value = want_bytes(value)
+        key = self.derive_key()
+        sig = self.algorithm.get_signature(key, value)
+        return base64_encode(sig)
+
+    def sign(self, value):
+        """Signs the given string."""
+        return want_bytes(value) + want_bytes(self.sep) + self.get_signature(value)
+
+    def verify_signature(self, value, sig):
+        """Verifies the signature for the given value."""
+        key = self.derive_key()
+        try:
+            sig = base64_decode(sig)
+        except Exception:
+            return False
+        return self.algorithm.verify_signature(key, value, sig)
+
+    def unsign(self, signed_value):
+        """Unsigns the given string."""
+        signed_value = want_bytes(signed_value)
+        sep = want_bytes(self.sep)
+        if sep not in signed_value:
+            raise BadSignature("No %r found in value" % self.sep)
+        value, sig = signed_value.rsplit(sep, 1)
+        if self.verify_signature(value, sig):
+            return value
+        raise BadSignature("Signature %r does not match" % sig, payload=value)
+
+    def validate(self, signed_value):
+        """Only validates the given signed value. Returns ``True`` if
+        the signature exists and is valid.
+        """
+        try:
+            self.unsign(signed_value)
+            return True
+        except BadSignature:
+            return False

python/itsdangerous/timed.py

@@ -0,0 +1,147 @@
+import time
+from datetime import datetime
+
+from ._compat import text_type
+from .encoding import base64_decode
+from .encoding import base64_encode
+from .encoding import bytes_to_int
+from .encoding import int_to_bytes
+from .encoding import want_bytes
+from .exc import BadSignature
+from .exc import BadTimeSignature
+from .exc import SignatureExpired
+from .serializer import Serializer
+from .signer import Signer
+
+
+class TimestampSigner(Signer):
+    """Works like the regular :class:`.Signer` but also records the time
+    of the signing and can be used to expire signatures. The
+    :meth:`unsign` method can raise :exc:`.SignatureExpired` if the
+    unsigning failed because the signature is expired.
+    """
+
+    def get_timestamp(self):
+        """Returns the current timestamp. The function must return an
+        integer.
+        """
+        return int(time.time())
+
+    def timestamp_to_datetime(self, ts):
+        """Used to convert the timestamp from :meth:`get_timestamp` into
+        a datetime object.
+        """
+        return datetime.utcfromtimestamp(ts)
+
+    def sign(self, value):
+        """Signs the given string and also attaches time information."""
+        value = want_bytes(value)
+        timestamp = base64_encode(int_to_bytes(self.get_timestamp()))
+        sep = want_bytes(self.sep)
+        value = value + sep + timestamp
+        return value + sep + self.get_signature(value)
+
+    def unsign(self, value, max_age=None, return_timestamp=False):
+        """Works like the regular :meth:`.Signer.unsign` but can also
+        validate the time. See the base docstring of the class for
+        the general behavior. If ``return_timestamp`` is ``True`` the
+        timestamp of the signature will be returned as a naive
+        :class:`datetime.datetime` object in UTC.
+        """
+        try:
+            result = Signer.unsign(self, value)
+            sig_error = None
+        except BadSignature as e:
+            sig_error = e
+            result = e.payload or b""
+        sep = want_bytes(self.sep)
+
+        # If there is no timestamp in the result there is something
+        # seriously wrong. In case there was a signature error, we raise
+        # that one directly, otherwise we have a weird situation in
+        # which we shouldn't have come except someone uses a time-based
+        # serializer on non-timestamp data, so catch that.
+        if sep not in result:
+            if sig_error:
+                raise sig_error
+            raise BadTimeSignature("timestamp missing", payload=result)
+
+        value, timestamp = result.rsplit(sep, 1)
+        try:
+            timestamp = bytes_to_int(base64_decode(timestamp))
+        except Exception:
+            timestamp = None
+
+        # Signature is *not* okay. Raise a proper error now that we have
+        # split the value and the timestamp.
+        if sig_error is not None:
+            raise BadTimeSignature(
+                text_type(sig_error), payload=value, date_signed=timestamp
+            )
+
+        # Signature was okay but the timestamp is actually not there or
+        # malformed. Should not happen, but we handle it anyway.
+        if timestamp is None:
+            raise BadTimeSignature("Malformed timestamp", payload=value)
+
+        # Check timestamp is not older than max_age
+        if max_age is not None:
+            age = self.get_timestamp() - timestamp
+            if age > max_age:
+                raise SignatureExpired(
+                    "Signature age %s > %s seconds" % (age, max_age),
+                    payload=value,
+                    date_signed=self.timestamp_to_datetime(timestamp),
+                )
+
+        if return_timestamp:
+            return value, self.timestamp_to_datetime(timestamp)
+        return value
+
+    def validate(self, signed_value, max_age=None):
+        """Only validates the given signed value. Returns ``True`` if
+        the signature exists and is valid."""
+        try:
+            self.unsign(signed_value, max_age=max_age)
+            return True
+        except BadSignature:
+            return False
+
+
+class TimedSerializer(Serializer):
+    """Uses :class:`TimestampSigner` instead of the default
+    :class:`.Signer`.
+    """
+
+    default_signer = TimestampSigner
+
+    def loads(self, s, max_age=None, return_timestamp=False, salt=None):
+        """Reverse of :meth:`dumps`, raises :exc:`.BadSignature` if the
+        signature validation fails. If a ``max_age`` is provided it will
+        ensure the signature is not older than that time in seconds. In
+        case the signature is outdated, :exc:`.SignatureExpired` is
+        raised. All arguments are forwarded to the signer's
+        :meth:`~TimestampSigner.unsign` method.
+        """
+        s = want_bytes(s)
+        last_exception = None
+        for signer in self.iter_unsigners(salt):
+            try:
+                base64d, timestamp = signer.unsign(s, max_age, return_timestamp=True)
+                payload = self.load_payload(base64d)
+                if return_timestamp:
+                    return payload, timestamp
+                return payload
+            # If we get a signature expired it means we could read the
+            # signature but it's invalid.  In that case we do not want to
+            # try the next signer.
+            except SignatureExpired:
+                raise
+            except BadSignature as err:
+                last_exception = err
+        raise last_exception
+
+    def loads_unsafe(self, s, max_age=None, salt=None):
+        load_kwargs = {"max_age": max_age}
+        load_payload_kwargs = {}
+        return self._loads_unsafe_impl(s, salt, load_kwargs, load_payload_kwargs)

python/itsdangerous/url_safe.py

@@ -0,0 +1,65 @@
+import zlib
+
+from ._json import _CompactJSON
+from .encoding import base64_decode
+from .encoding import base64_encode
+from .exc import BadPayload
+from .serializer import Serializer
+from .timed import TimedSerializer
+
+
+class URLSafeSerializerMixin(object):
+    """Mixed in with a regular serializer it will attempt to zlib
+    compress the string to make it shorter if necessary. It will also
+    base64 encode the string so that it can safely be placed in a URL.
+    """
+
+    default_serializer = _CompactJSON
+
+    def load_payload(self, payload, *args, **kwargs):
+        decompress = False
+        if payload.startswith(b"."):
+            payload = payload[1:]
+            decompress = True
+        try:
+            json = base64_decode(payload)
+        except Exception as e:
+            raise BadPayload(
+                "Could not base64 decode the payload because of an exception",
+                original_error=e,
+            )
+        if decompress:
+            try:
+                json = zlib.decompress(json)
+            except Exception as e:
+                raise BadPayload(
+                    "Could not zlib decompress the payload before decoding the payload",
+                    original_error=e,
+                )
+        return super(URLSafeSerializerMixin, self).load_payload(json, *args, **kwargs)
+
+    def dump_payload(self, obj):
+        json = super(URLSafeSerializerMixin, self).dump_payload(obj)
+        is_compressed = False
+        compressed = zlib.compress(json)
+        if len(compressed) < (len(json) - 1):
+            json = compressed
+            is_compressed = True
+        base64d = base64_encode(json)
+        if is_compressed:
+            base64d = b"." + base64d
+        return base64d
+
+
+class URLSafeSerializer(URLSafeSerializerMixin, Serializer):
+    """Works like :class:`.Serializer` but dumps and loads into a URL
+    safe string consisting of the upper and lowercase character of the
+    alphabet as well as ``'_'``, ``'-'`` and ``'.'``.
+    """
+
+
+class URLSafeTimedSerializer(URLSafeSerializerMixin, TimedSerializer):
+    """Works like :class:`.TimedSerializer` but dumps and loads into a
+    URL safe string consisting of the upper and lowercase character of
+    the alphabet as well as ``'_'``, ``'-'`` and ``'.'``.
+    """

python/jinja2/__init__.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2
+    ~~~~~~
+
+    Jinja2 is a template engine written in pure Python.  It provides a
+    Django inspired non-XML syntax but supports inline expressions and
+    an optional sandboxed environment.
+
+    Nutshell
+    --------
+
+    Here a small example of a Jinja2 template::
+
+        {% extends 'base.html' %}
+        {% block title %}Memberlist{% endblock %}
+        {% block content %}
+          <ul>
+          {% for user in users %}
+            <li><a href="{{ user.url }}">{{ user.username }}</a></li>
+          {% endfor %}
+          </ul>
+        {% endblock %}
+
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+__docformat__ = 'restructuredtext en'
+__version__ = '2.10.1'
+
+# high level interface
+from jinja2.environment import Environment, Template
+
+# loaders
+from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
+     DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \
+     ModuleLoader
+
+# bytecode caches
+from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \
+     MemcachedBytecodeCache
+
+# undefined types
+from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \
+     make_logging_undefined
+
+# exceptions
+from jinja2.exceptions import TemplateError, UndefinedError, \
+     TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \
+     TemplateAssertionError, TemplateRuntimeError
+
+# decorators and public utilities
+from jinja2.filters import environmentfilter, contextfilter, \
+     evalcontextfilter
+from jinja2.utils import Markup, escape, clear_caches, \
+     environmentfunction, evalcontextfunction, contextfunction, \
+     is_undefined, select_autoescape
+
+__all__ = [
+    'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
+    'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader',
+    'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache',
+    'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined',
+    'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound',
+    'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError',
+    'TemplateRuntimeError',
+    'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape',
+    'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined',
+    'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined',
+    'select_autoescape',
+]
+
+
+def _patch_async():
+    from jinja2.utils import have_async_gen
+    if have_async_gen:
+        from jinja2.asyncsupport import patch_all
+        patch_all()
+
+
+_patch_async()
+del _patch_async

python/jinja2/_compat.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2._compat
+    ~~~~~~~~~~~~~~
+
+    Some py2/py3 compatibility support based on a stripped down
+    version of six so we don't have to depend on a specific version
+    of it.
+
+    :copyright: Copyright 2013 by the Jinja team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import sys
+
+PY2 = sys.version_info[0] == 2
+PYPY = hasattr(sys, 'pypy_translation_info')
+_identity = lambda x: x
+
+
+if not PY2:
+    unichr = chr
+    range_type = range
+    text_type = str
+    string_types = (str,)
+    integer_types = (int,)
+
+    iterkeys = lambda d: iter(d.keys())
+    itervalues = lambda d: iter(d.values())
+    iteritems = lambda d: iter(d.items())
+
+    import pickle
+    from io import BytesIO, StringIO
+    NativeStringIO = StringIO
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+    ifilter = filter
+    imap = map
+    izip = zip
+    intern = sys.intern
+
+    implements_iterator = _identity
+    implements_to_string = _identity
+    encode_filename = _identity
+
+else:
+    unichr = unichr
+    text_type = unicode
+    range_type = xrange
+    string_types = (str, unicode)
+    integer_types = (int, long)
+
+    iterkeys = lambda d: d.iterkeys()
+    itervalues = lambda d: d.itervalues()
+    iteritems = lambda d: d.iteritems()
+
+    import cPickle as pickle
+    from cStringIO import StringIO as BytesIO, StringIO
+    NativeStringIO = BytesIO
+
+    exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
+
+    from itertools import imap, izip, ifilter
+    intern = intern
+
+    def implements_iterator(cls):
+        cls.next = cls.__next__
+        del cls.__next__
+        return cls
+
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+
+    def encode_filename(filename):
+        if isinstance(filename, unicode):
+            return filename.encode('utf-8')
+        return filename
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a
+    # dummy metaclass for one level of class instantiation that replaces
+    # itself with the actual metaclass.
+    class metaclass(type):
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+try:
+    from urllib.parse import quote_from_bytes as url_quote
+except ImportError:
+    from urllib import quote as url_quote

python/jinja2/_identifier.py

@@ -0,0 +1,2 @@
+# generated by scripts/generate_identifier_pattern.py
+pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯'

python/jinja2/asyncfilters.py

@@ -0,0 +1,146 @@
+from functools import wraps
+
+from jinja2.asyncsupport import auto_aiter
+from jinja2 import filters
+
+
+async def auto_to_seq(value):
+    seq = []
+    if hasattr(value, '__aiter__'):
+        async for item in value:
+            seq.append(item)
+    else:
+        for item in value:
+            seq.append(item)
+    return seq
+
+
+async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
+    seq, func = filters.prepare_select_or_reject(
+        args, kwargs, modfunc, lookup_attr)
+    if seq:
+        async for item in auto_aiter(seq):
+            if func(item):
+                yield item
+
+
+def dualfilter(normal_filter, async_filter):
+    wrap_evalctx = False
+    if getattr(normal_filter, 'environmentfilter', False):
+        is_async = lambda args: args[0].is_async
+        wrap_evalctx = False
+    else:
+        if not getattr(normal_filter, 'evalcontextfilter', False) and \
+           not getattr(normal_filter, 'contextfilter', False):
+            wrap_evalctx = True
+        is_async = lambda args: args[0].environment.is_async
+
+    @wraps(normal_filter)
+    def wrapper(*args, **kwargs):
+        b = is_async(args)
+        if wrap_evalctx:
+            args = args[1:]
+        if b:
+            return async_filter(*args, **kwargs)
+        return normal_filter(*args, **kwargs)
+
+    if wrap_evalctx:
+        wrapper.evalcontextfilter = True
+
+    wrapper.asyncfiltervariant = True
+
+    return wrapper
+
+
+def asyncfiltervariant(original):
+    def decorator(f):
+        return dualfilter(original, f)
+    return decorator
+
+
+@asyncfiltervariant(filters.do_first)
+async def do_first(environment, seq):
+    try:
+        return await auto_aiter(seq).__anext__()
+    except StopAsyncIteration:
+        return environment.undefined('No first item, sequence was empty.')
+
+
+@asyncfiltervariant(filters.do_groupby)
+async def do_groupby(environment, value, attribute):
+    expr = filters.make_attrgetter(environment, attribute)
+    return [filters._GroupTuple(key, await auto_to_seq(values))
+            for key, values in filters.groupby(sorted(
+                await auto_to_seq(value), key=expr), expr)]
+
+
+@asyncfiltervariant(filters.do_join)
+async def do_join(eval_ctx, value, d=u'', attribute=None):
+    return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute)
+
+
+@asyncfiltervariant(filters.do_list)
+async def do_list(value):
+    return await auto_to_seq(value)
+
+
+@asyncfiltervariant(filters.do_reject)
+async def do_reject(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: not x, False)
+
+
+@asyncfiltervariant(filters.do_rejectattr)
+async def do_rejectattr(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: not x, True)
+
+
+@asyncfiltervariant(filters.do_select)
+async def do_select(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: x, False)
+
+
+@asyncfiltervariant(filters.do_selectattr)
+async def do_selectattr(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: x, True)
+
+
+@asyncfiltervariant(filters.do_map)
+async def do_map(*args, **kwargs):
+    seq, func = filters.prepare_map(args, kwargs)
+    if seq:
+        async for item in auto_aiter(seq):
+            yield func(item)
+
+
+@asyncfiltervariant(filters.do_sum)
+async def do_sum(environment, iterable, attribute=None, start=0):
+    rv = start
+    if attribute is not None:
+        func = filters.make_attrgetter(environment, attribute)
+    else:
+        func = lambda x: x
+    async for item in auto_aiter(iterable):
+        rv += func(item)
+    return rv
+
+
+@asyncfiltervariant(filters.do_slice)
+async def do_slice(value, slices, fill_with=None):
+    return filters.do_slice(await auto_to_seq(value), slices, fill_with)
+
+
+ASYNC_FILTERS = {
+    'first':        do_first,
+    'groupby':      do_groupby,
+    'join':         do_join,
+    'list':         do_list,
+    # we intentionally do not support do_last because that would be
+    # ridiculous
+    'reject':       do_reject,
+    'rejectattr':   do_rejectattr,
+    'map':          do_map,
+    'select':       do_select,
+    'selectattr':   do_selectattr,
+    'sum':          do_sum,
+    'slice':        do_slice,
+}

python/jinja2/asyncsupport.py

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.asyncsupport
+    ~~~~~~~~~~~~~~~~~~~
+
+    Has all the code for async support which is implemented as a patch
+    for supported Python versions.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+import asyncio
+import inspect
+from functools import update_wrapper
+
+from jinja2.utils import concat, internalcode, Markup
+from jinja2.environment import TemplateModule
+from jinja2.runtime import LoopContextBase, _last_iteration
+
+
+async def concat_async(async_gen):
+    rv = []
+    async def collect():
+        async for event in async_gen:
+            rv.append(event)
+    await collect()
+    return concat(rv)
+
+
+async def generate_async(self, *args, **kwargs):
+    vars = dict(*args, **kwargs)
+    try:
+        async for event in self.root_render_func(self.new_context(vars)):
+            yield event
+    except Exception:
+        exc_info = sys.exc_info()
+    else:
+        return
+    yield self.environment.handle_exception(exc_info, True)
+
+
+def wrap_generate_func(original_generate):
+    def _convert_generator(self, loop, args, kwargs):
+        async_gen = self.generate_async(*args, **kwargs)
+        try:
+            while 1:
+                yield loop.run_until_complete(async_gen.__anext__())
+        except StopAsyncIteration:
+            pass
+    def generate(self, *args, **kwargs):
+        if not self.environment.is_async:
+            return original_generate(self, *args, **kwargs)
+        return _convert_generator(self, asyncio.get_event_loop(), args, kwargs)
+    return update_wrapper(generate, original_generate)
+
+
+async def render_async(self, *args, **kwargs):
+    if not self.environment.is_async:
+        raise RuntimeError('The environment was not created with async mode '
+                           'enabled.')
+
+    vars = dict(*args, **kwargs)
+    ctx = self.new_context(vars)
+
+    try:
+        return await concat_async(self.root_render_func(ctx))
+    except Exception:
+        exc_info = sys.exc_info()
+    return self.environment.handle_exception(exc_info, True)
+
+
+def wrap_render_func(original_render):
+    def render(self, *args, **kwargs):
+        if not self.environment.is_async:
+            return original_render(self, *args, **kwargs)
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(self.render_async(*args, **kwargs))
+    return update_wrapper(render, original_render)
+
+
+def wrap_block_reference_call(original_call):
+    @internalcode
+    async def async_call(self):
+        rv = await concat_async(self._stack[self._depth](self._context))
+        if self._context.eval_ctx.autoescape:
+            rv = Markup(rv)
+        return rv
+
+    @internalcode
+    def __call__(self):
+        if not self._context.environment.is_async:
+            return original_call(self)
+        return async_call(self)
+
+    return update_wrapper(__call__, original_call)
+
+
+def wrap_macro_invoke(original_invoke):
+    @internalcode
+    async def async_invoke(self, arguments, autoescape):
+        rv = await self._func(*arguments)
+        if autoescape:
+            rv = Markup(rv)
+        return rv
+
+    @internalcode
+    def _invoke(self, arguments, autoescape):
+        if not self._environment.is_async:
+            return original_invoke(self, arguments, autoescape)
+        return async_invoke(self, arguments, autoescape)
+    return update_wrapper(_invoke, original_invoke)
+
+
+@internalcode
+async def get_default_module_async(self):
+    if self._module is not None:
+        return self._module
+    self._module = rv = await self.make_module_async()
+    return rv
+
+
+def wrap_default_module(original_default_module):
+    @internalcode
+    def _get_default_module(self):
+        if self.environment.is_async:
+            raise RuntimeError('Template module attribute is unavailable '
+                               'in async mode')
+        return original_default_module(self)
+    return _get_default_module
+
+
+async def make_module_async(self, vars=None, shared=False, locals=None):
+    context = self.new_context(vars, shared, locals)
+    body_stream = []
+    async for item in self.root_render_func(context):
+        body_stream.append(item)
+    return TemplateModule(self, context, body_stream)
+
+
+def patch_template():
+    from jinja2 import Template
+    Template.generate = wrap_generate_func(Template.generate)
+    Template.generate_async = update_wrapper(
+        generate_async, Template.generate_async)
+    Template.render_async = update_wrapper(
+        render_async, Template.render_async)
+    Template.render = wrap_render_func(Template.render)
+    Template._get_default_module = wrap_default_module(
+        Template._get_default_module)
+    Template._get_default_module_async = get_default_module_async
+    Template.make_module_async = update_wrapper(
+        make_module_async, Template.make_module_async)
+
+
+def patch_runtime():
+    from jinja2.runtime import BlockReference, Macro
+    BlockReference.__call__ = wrap_block_reference_call(
+        BlockReference.__call__)
+    Macro._invoke = wrap_macro_invoke(Macro._invoke)
+
+
+def patch_filters():
+    from jinja2.filters import FILTERS
+    from jinja2.asyncfilters import ASYNC_FILTERS
+    FILTERS.update(ASYNC_FILTERS)
+
+
+def patch_all():
+    patch_template()
+    patch_runtime()
+    patch_filters()
+
+
+async def auto_await(value):
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+async def auto_aiter(iterable):
+    if hasattr(iterable, '__aiter__'):
+        async for item in iterable:
+            yield item
+        return
+    for item in iterable:
+        yield item
+
+
+class AsyncLoopContext(LoopContextBase):
+
+    def __init__(self, async_iterator, undefined, after, length, recurse=None,
+                 depth0=0):
+        LoopContextBase.__init__(self, undefined, recurse, depth0)
+        self._async_iterator = async_iterator
+        self._after = after
+        self._length = length
+
+    @property
+    def length(self):
+        if self._length is None:
+            raise TypeError('Loop length for some iterators cannot be '
+                            'lazily calculated in async mode')
+        return self._length
+
+    def __aiter__(self):
+        return AsyncLoopContextIterator(self)
+
+
+class AsyncLoopContextIterator(object):
+    __slots__ = ('context',)
+
+    def __init__(self, context):
+        self.context = context
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        ctx = self.context
+        ctx.index0 += 1
+        if ctx._after is _last_iteration:
+            raise StopAsyncIteration()
+        ctx._before = ctx._current
+        ctx._current = ctx._after
+        try:
+            ctx._after = await ctx._async_iterator.__anext__()
+        except StopAsyncIteration:
+            ctx._after = _last_iteration
+        return ctx._current, ctx
+
+
+async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0):
+    # Length is more complicated and less efficient in async mode.  The
+    # reason for this is that we cannot know if length will be used
+    # upfront but because length is a property we cannot lazily execute it
+    # later.  This means that we need to buffer it up and measure :(
+    #
+    # We however only do this for actual iterators, not for async
+    # iterators as blocking here does not seem like the best idea in the
+    # world.
+    try:
+        length = len(iterable)
+    except (TypeError, AttributeError):
+        if not hasattr(iterable, '__aiter__'):
+            iterable = tuple(iterable)
+            length = len(iterable)
+        else:
+            length = None
+    async_iterator = auto_aiter(iterable)
+    try:
+        after = await async_iterator.__anext__()
+    except StopAsyncIteration:
+        after = _last_iteration
+    return AsyncLoopContext(async_iterator, undefined, after, length, recurse,
+                            depth0)

python/jinja2/bccache.py

@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.bccache
+    ~~~~~~~~~~~~~~
+
+    This module implements the bytecode cache system Jinja is optionally
+    using.  This is useful if you have very complex template situations and
+    the compiliation of all those templates slow down your application too
+    much.
+
+    Situations where this is useful are often forking web applications that
+    are initialized on the first request.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+from os import path, listdir
+import os
+import sys
+import stat
+import errno
+import marshal
+import tempfile
+import fnmatch
+from hashlib import sha1
+from jinja2.utils import open_if_exists
+from jinja2._compat import BytesIO, pickle, PY2, text_type
+
+
+# marshal works better on 3.x, one hack less required
+if not PY2:
+    marshal_dump = marshal.dump
+    marshal_load = marshal.load
+else:
+
+    def marshal_dump(code, f):
+        if isinstance(f, file):
+            marshal.dump(code, f)
+        else:
+            f.write(marshal.dumps(code))
+
+    def marshal_load(f):
+        if isinstance(f, file):
+            return marshal.load(f)
+        return marshal.loads(f.read())
+
+
+bc_version = 3
+
+# magic version used to only change with new jinja versions.  With 2.6
+# we change this to also take Python version changes into account.  The
+# reason for this is that Python tends to segfault if fed earlier bytecode
+# versions because someone thought it would be a good idea to reuse opcodes
+# or make Python incompatible with earlier versions.
+bc_magic = 'j2'.encode('ascii') + \
+    pickle.dumps(bc_version, 2) + \
+    pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
+
+
+class Bucket(object):
+    """Buckets are used to store the bytecode for one template.  It's created
+    and initialized by the bytecode cache and passed to the loading functions.
+
+    The buckets get an internal checksum from the cache assigned and use this
+    to automatically reject outdated cache material.  Individual bytecode
+    cache subclasses don't have to care about cache invalidation.
+    """
+
+    def __init__(self, environment, key, checksum):
+        self.environment = environment
+        self.key = key
+        self.checksum = checksum
+        self.reset()
+
+    def reset(self):
+        """Resets the bucket (unloads the bytecode)."""
+        self.code = None
+
+    def load_bytecode(self, f):
+        """Loads bytecode from a file or file like object."""
+        # make sure the magic header is correct
+        magic = f.read(len(bc_magic))
+        if magic != bc_magic:
+            self.reset()
+            return
+        # the source code of the file changed, we need to reload
+        checksum = pickle.load(f)
+        if self.checksum != checksum:
+            self.reset()
+            return
+        # if marshal_load fails then we need to reload
+        try:
+            self.code = marshal_load(f)
+        except (EOFError, ValueError, TypeError):
+            self.reset()
+            return
+
+    def write_bytecode(self, f):
+        """Dump the bytecode into the file or file like object passed."""
+        if self.code is None:
+            raise TypeError('can\'t write empty bucket')
+        f.write(bc_magic)
+        pickle.dump(self.checksum, f, 2)
+        marshal_dump(self.code, f)
+
+    def bytecode_from_string(self, string):
+        """Load bytecode from a string."""
+        self.load_bytecode(BytesIO(string))
+
+    def bytecode_to_string(self):
+        """Return the bytecode as string."""
+        out = BytesIO()
+        self.write_bytecode(out)
+        return out.getvalue()
+
+
+class BytecodeCache(object):
+    """To implement your own bytecode cache you have to subclass this class
+    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
+    these methods are passed a :class:`~jinja2.bccache.Bucket`.
+
+    A very basic bytecode cache that saves the bytecode on the file system::
+
+        from os import path
+
+        class MyCache(BytecodeCache):
+
+            def __init__(self, directory):
+                self.directory = directory
+
+            def load_bytecode(self, bucket):
+                filename = path.join(self.directory, bucket.key)
+                if path.exists(filename):
+                    with open(filename, 'rb') as f:
+                        bucket.load_bytecode(f)
+
+            def dump_bytecode(self, bucket):
+                filename = path.join(self.directory, bucket.key)
+                with open(filename, 'wb') as f:
+                    bucket.write_bytecode(f)
+
+    A more advanced version of a filesystem based bytecode cache is part of
+    Jinja2.
+    """
+
+    def load_bytecode(self, bucket):
+        """Subclasses have to override this method to load bytecode into a
+        bucket.  If they are not able to find code in the cache for the
+        bucket, it must not do anything.
+        """
+        raise NotImplementedError()
+
+    def dump_bytecode(self, bucket):
+        """Subclasses have to override this method to write the bytecode
+        from a bucket back to the cache.  If it unable to do so it must not
+        fail silently but raise an exception.
+        """
+        raise NotImplementedError()
+
+    def clear(self):
+        """Clears the cache.  This method is not used by Jinja2 but should be
+        implemented to allow applications to clear the bytecode cache used
+        by a particular environment.
+        """
+
+    def get_cache_key(self, name, filename=None):
+        """Returns the unique hash key for this template name."""
+        hash = sha1(name.encode('utf-8'))
+        if filename is not None:
+            filename = '|' + filename
+            if isinstance(filename, text_type):
+                filename = filename.encode('utf-8')
+            hash.update(filename)
+        return hash.hexdigest()
+
+    def get_source_checksum(self, source):
+        """Returns a checksum for the source."""
+        return sha1(source.encode('utf-8')).hexdigest()
+
+    def get_bucket(self, environment, name, filename, source):
+        """Return a cache bucket for the given template.  All arguments are
+        mandatory but filename may be `None`.
+        """
+        key = self.get_cache_key(name, filename)
+        checksum = self.get_source_checksum(source)
+        bucket = Bucket(environment, key, checksum)
+        self.load_bytecode(bucket)
+        return bucket
+
+    def set_bucket(self, bucket):
+        """Put the bucket into the cache."""
+        self.dump_bytecode(bucket)
+
+
+class FileSystemBytecodeCache(BytecodeCache):
+    """A bytecode cache that stores bytecode on the filesystem.  It accepts
+    two arguments: The directory where the cache items are stored and a
+    pattern string that is used to build the filename.
+
+    If no directory is specified a default cache directory is selected.  On
+    Windows the user's temp directory is used, on UNIX systems a directory
+    is created for the user in the system temp directory.
+
+    The pattern can be used to have multiple separate caches operate on the
+    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
+    is replaced with the cache key.
+
+    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
+
+    This bytecode cache supports clearing of the cache using the clear method.
+    """
+
+    def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
+        if directory is None:
+            directory = self._get_default_cache_dir()
+        self.directory = directory
+        self.pattern = pattern
+
+    def _get_default_cache_dir(self):
+        def _unsafe_dir():
+            raise RuntimeError('Cannot determine safe temp directory.  You '
+                               'need to explicitly provide one.')
+
+        tmpdir = tempfile.gettempdir()
+
+        # On windows the temporary directory is used specific unless
+        # explicitly forced otherwise.  We can just use that.
+        if os.name == 'nt':
+            return tmpdir
+        if not hasattr(os, 'getuid'):
+            _unsafe_dir()
+
+        dirname = '_jinja2-cache-%d' % os.getuid()
+        actual_dir = os.path.join(tmpdir, dirname)
+
+        try:
+            os.mkdir(actual_dir, stat.S_IRWXU)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        try:
+            os.chmod(actual_dir, stat.S_IRWXU)
+            actual_dir_stat = os.lstat(actual_dir)
+            if actual_dir_stat.st_uid != os.getuid() \
+               or not stat.S_ISDIR(actual_dir_stat.st_mode) \
+               or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
+                _unsafe_dir()
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        actual_dir_stat = os.lstat(actual_dir)
+        if actual_dir_stat.st_uid != os.getuid() \
+           or not stat.S_ISDIR(actual_dir_stat.st_mode) \
+           or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
+            _unsafe_dir()
+
+        return actual_dir
+
+    def _get_cache_filename(self, bucket):
+        return path.join(self.directory, self.pattern % bucket.key)
+
+    def load_bytecode(self, bucket):
+        f = open_if_exists(self._get_cache_filename(bucket), 'rb')
+        if f is not None:
+            try:
+                bucket.load_bytecode(f)
+            finally:
+                f.close()
+
+    def dump_bytecode(self, bucket):
+        f = open(self._get_cache_filename(bucket), 'wb')
+        try:
+            bucket.write_bytecode(f)
+        finally:
+            f.close()
+
+    def clear(self):
+        # imported lazily here because google app-engine doesn't support
+        # write access on the file system and the function does not exist
+        # normally.
+        from os import remove
+        files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
+        for filename in files:
+            try:
+                remove(path.join(self.directory, filename))
+            except OSError:
+                pass
+
+
+class MemcachedBytecodeCache(BytecodeCache):
+    """This class implements a bytecode cache that uses a memcache cache for
+    storing the information.  It does not enforce a specific memcache library
+    (tummy's memcache or cmemcache) but will accept any class that provides
+    the minimal interface required.
+
+    Libraries compatible with this class:
+
+    -   `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
+    -   `python-memcached <https://www.tummy.com/Community/software/python-memcached/>`_
+    -   `cmemcache <http://gijsbert.org/cmemcache/>`_
+
+    (Unfortunately the django cache interface is not compatible because it
+    does not support storing binary data, only unicode.  You can however pass
+    the underlying cache client to the bytecode cache which is available
+    as `django.core.cache.cache._client`.)
+
+    The minimal interface for the client passed to the constructor is this:
+
+    .. class:: MinimalClientInterface
+
+        .. method:: set(key, value[, timeout])
+
+            Stores the bytecode in the cache.  `value` is a string and
+            `timeout` the timeout of the key.  If timeout is not provided
+            a default timeout or no timeout should be assumed, if it's
+            provided it's an integer with the number of seconds the cache
+            item should exist.
+
+        .. method:: get(key)
+
+            Returns the value for the cache key.  If the item does not
+            exist in the cache the return value must be `None`.
+
+    The other arguments to the constructor are the prefix for all keys that
+    is added before the actual cache key and the timeout for the bytecode in
+    the cache system.  We recommend a high (or no) timeout.
+
+    This bytecode cache does not support clearing of used items in the cache.
+    The clear method is a no-operation function.
+
+    .. versionadded:: 2.7
+       Added support for ignoring memcache errors through the
+       `ignore_memcache_errors` parameter.
+    """
+
+    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
+                 ignore_memcache_errors=True):
+        self.client = client
+        self.prefix = prefix
+        self.timeout = timeout
+        self.ignore_memcache_errors = ignore_memcache_errors
+
+    def load_bytecode(self, bucket):
+        try:
+            code = self.client.get(self.prefix + bucket.key)
+        except Exception:
+            if not self.ignore_memcache_errors:
+                raise
+            code = None
+        if code is not None:
+            bucket.bytecode_from_string(code)
+
+    def dump_bytecode(self, bucket):
+        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
+        if self.timeout is not None:
+            args += (self.timeout,)
+        try:
+            self.client.set(*args)
+        except Exception:
+            if not self.ignore_memcache_errors:
+                raise

python/jinja2/constants.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja.constants
+    ~~~~~~~~~~~~~~~
+
+    Various constants.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+#: list of lorem ipsum words used by the lipsum() helper function
+LOREM_IPSUM_WORDS = u'''\
+a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at
+auctor augue bibendum blandit class commodo condimentum congue consectetuer
+consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus
+diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend
+elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames
+faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac
+hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum
+justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem
+luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie
+mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non
+nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque
+penatibus per pharetra phasellus placerat platea porta porttitor posuere
+potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus
+ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit
+sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor
+tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices
+ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus
+viverra volutpat vulputate'''

python/jinja2/debug.py

@@ -0,0 +1,372 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.debug
+    ~~~~~~~~~~~~
+
+    Implements the debug interface for Jinja.  This module does some pretty
+    ugly stuff with the Python traceback system in order to achieve tracebacks
+    with correct line numbers, locals and contents.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+import traceback
+from types import TracebackType, CodeType
+from jinja2.utils import missing, internal_code
+from jinja2.exceptions import TemplateSyntaxError
+from jinja2._compat import iteritems, reraise, PY2
+
+# on pypy we can take advantage of transparent proxies
+try:
+    from __pypy__ import tproxy
+except ImportError:
+    tproxy = None
+
+
+# how does the raise helper look like?
+try:
+    exec("raise TypeError, 'foo'")
+except SyntaxError:
+    raise_helper = 'raise __jinja_exception__[1]'
+except TypeError:
+    raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
+
+
+class TracebackFrameProxy(object):
+    """Proxies a traceback frame."""
+
+    def __init__(self, tb):
+        self.tb = tb
+        self._tb_next = None
+
+    @property
+    def tb_next(self):
+        return self._tb_next
+
+    def set_next(self, next):
+        if tb_set_next is not None:
+            try:
+                tb_set_next(self.tb, next and next.tb or None)
+            except Exception:
+                # this function can fail due to all the hackery it does
+                # on various python implementations.  We just catch errors
+                # down and ignore them if necessary.
+                pass
+        self._tb_next = next
+
+    @property
+    def is_jinja_frame(self):
+        return '__jinja_template__' in self.tb.tb_frame.f_globals
+
+    def __getattr__(self, name):
+        return getattr(self.tb, name)
+
+
+def make_frame_proxy(frame):
+    proxy = TracebackFrameProxy(frame)
+    if tproxy is None:
+        return proxy
+    def operation_handler(operation, *args, **kwargs):
+        if operation in ('__getattribute__', '__getattr__'):
+            return getattr(proxy, args[0])
+        elif operation == '__setattr__':
+            proxy.__setattr__(*args, **kwargs)
+        else:
+            return getattr(proxy, operation)(*args, **kwargs)
+    return tproxy(TracebackType, operation_handler)
+
+
+class ProcessedTraceback(object):
+    """Holds a Jinja preprocessed traceback for printing or reraising."""
+
+    def __init__(self, exc_type, exc_value, frames):
+        assert frames, 'no frames for this traceback?'
+        self.exc_type = exc_type
+        self.exc_value = exc_value
+        self.frames = frames
+
+        # newly concatenate the frames (which are proxies)
+        prev_tb = None
+        for tb in self.frames:
+            if prev_tb is not None:
+                prev_tb.set_next(tb)
+            prev_tb = tb
+        prev_tb.set_next(None)
+
+    def render_as_text(self, limit=None):
+        """Return a string with the traceback."""
+        lines = traceback.format_exception(self.exc_type, self.exc_value,
+                                           self.frames[0], limit=limit)
+        return ''.join(lines).rstrip()
+
+    def render_as_html(self, full=False):
+        """Return a unicode string with the traceback as rendered HTML."""
+        from jinja2.debugrenderer import render_traceback
+        return u'%s\n\n<!--\n%s\n-->' % (
+            render_traceback(self, full=full),
+            self.render_as_text().decode('utf-8', 'replace')
+        )
+
+    @property
+    def is_template_syntax_error(self):
+        """`True` if this is a template syntax error."""
+        return isinstance(self.exc_value, TemplateSyntaxError)
+
+    @property
+    def exc_info(self):
+        """Exception info tuple with a proxy around the frame objects."""
+        return self.exc_type, self.exc_value, self.frames[0]
+
+    @property
+    def standard_exc_info(self):
+        """Standard python exc_info for re-raising"""
+        tb = self.frames[0]
+        # the frame will be an actual traceback (or transparent proxy) if
+        # we are on pypy or a python implementation with support for tproxy
+        if type(tb) is not TracebackType:
+            tb = tb.tb
+        return self.exc_type, self.exc_value, tb
+
+
+def make_traceback(exc_info, source_hint=None):
+    """Creates a processed traceback object from the exc_info."""
+    exc_type, exc_value, tb = exc_info
+    if isinstance(exc_value, TemplateSyntaxError):
+        exc_info = translate_syntax_error(exc_value, source_hint)
+        initial_skip = 0
+    else:
+        initial_skip = 1
+    return translate_exception(exc_info, initial_skip)
+
+
+def translate_syntax_error(error, source=None):
+    """Rewrites a syntax error to please traceback systems."""
+    error.source = source
+    error.translated = True
+    exc_info = (error.__class__, error, None)
+    filename = error.filename
+    if filename is None:
+        filename = '<unknown>'
+    return fake_exc_info(exc_info, filename, error.lineno)
+
+
+def translate_exception(exc_info, initial_skip=0):
+    """If passed an exc_info it will automatically rewrite the exceptions
+    all the way down to the correct line numbers and frames.
+    """
+    tb = exc_info[2]
+    frames = []
+
+    # skip some internal frames if wanted
+    for x in range(initial_skip):
+        if tb is not None:
+            tb = tb.tb_next
+    initial_tb = tb
+
+    while tb is not None:
+        # skip frames decorated with @internalcode.  These are internal
+        # calls we can't avoid and that are useless in template debugging
+        # output.
+        if tb.tb_frame.f_code in internal_code:
+            tb = tb.tb_next
+            continue
+
+        # save a reference to the next frame if we override the current
+        # one with a faked one.
+        next = tb.tb_next
+
+        # fake template exceptions
+        template = tb.tb_frame.f_globals.get('__jinja_template__')
+        if template is not None:
+            lineno = template.get_corresponding_lineno(tb.tb_lineno)
+            tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
+                               lineno)[2]
+
+        frames.append(make_frame_proxy(tb))
+        tb = next
+
+    # if we don't have any exceptions in the frames left, we have to
+    # reraise it unchanged.
+    # XXX: can we backup here?  when could this happen?
+    if not frames:
+        reraise(exc_info[0], exc_info[1], exc_info[2])
+
+    return ProcessedTraceback(exc_info[0], exc_info[1], frames)
+
+
+def get_jinja_locals(real_locals):
+    ctx = real_locals.get('context')
+    if ctx:
+        locals = ctx.get_all().copy()
+    else:
+        locals = {}
+
+    local_overrides = {}
+
+    for name, value in iteritems(real_locals):
+        if not name.startswith('l_') or value is missing:
+            continue
+        try:
+            _, depth, name = name.split('_', 2)
+            depth = int(depth)
+        except ValueError:
+            continue
+        cur_depth = local_overrides.get(name, (-1,))[0]
+        if cur_depth < depth:
+            local_overrides[name] = (depth, value)
+
+    for name, (_, value) in iteritems(local_overrides):
+        if value is missing:
+            locals.pop(name, None)
+        else:
+            locals[name] = value
+
+    return locals
+
+
+def fake_exc_info(exc_info, filename, lineno):
+    """Helper for `translate_exception`."""
+    exc_type, exc_value, tb = exc_info
+
+    # figure the real context out
+    if tb is not None:
+        locals = get_jinja_locals(tb.tb_frame.f_locals)
+
+        # if there is a local called __jinja_exception__, we get
+        # rid of it to not break the debug functionality.
+        locals.pop('__jinja_exception__', None)
+    else:
+        locals = {}
+
+    # assamble fake globals we need
+    globals = {
+        '__name__':             filename,
+        '__file__':             filename,
+        '__jinja_exception__':  exc_info[:2],
+
+        # we don't want to keep the reference to the template around
+        # to not cause circular dependencies, but we mark it as Jinja
+        # frame for the ProcessedTraceback
+        '__jinja_template__':   None
+    }
+
+    # and fake the exception
+    code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
+
+    # if it's possible, change the name of the code.  This won't work
+    # on some python environments such as google appengine
+    try:
+        if tb is None:
+            location = 'template'
+        else:
+            function = tb.tb_frame.f_code.co_name
+            if function == 'root':
+                location = 'top-level template code'
+            elif function.startswith('block_'):
+                location = 'block "%s"' % function[6:]
+            else:
+                location = 'template'
+
+        if PY2:
+            code = CodeType(0, code.co_nlocals, code.co_stacksize,
+                            code.co_flags, code.co_code, code.co_consts,
+                            code.co_names, code.co_varnames, filename,
+                            location, code.co_firstlineno,
+                            code.co_lnotab, (), ())
+        else:
+            code = CodeType(0, code.co_kwonlyargcount,
+                            code.co_nlocals, code.co_stacksize,
+                            code.co_flags, code.co_code, code.co_consts,
+                            code.co_names, code.co_varnames, filename,
+                            location, code.co_firstlineno,
+                            code.co_lnotab, (), ())
+    except Exception as e:
+        pass
+
+    # execute the code and catch the new traceback
+    try:
+        exec(code, globals, locals)
+    except:
+        exc_info = sys.exc_info()
+        new_tb = exc_info[2].tb_next
+
+    # return without this frame
+    return exc_info[:2] + (new_tb,)
+
+
+def _init_ugly_crap():
+    """This function implements a few ugly things so that we can patch the
+    traceback objects.  The function returned allows resetting `tb_next` on
+    any python traceback object.  Do not attempt to use this on non cpython
+    interpreters
+    """
+    import ctypes
+    from types import TracebackType
+
+    if PY2:
+        # figure out size of _Py_ssize_t for Python 2:
+        if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
+            _Py_ssize_t = ctypes.c_int64
+        else:
+            _Py_ssize_t = ctypes.c_int
+    else:
+        # platform ssize_t on Python 3
+        _Py_ssize_t = ctypes.c_ssize_t
+
+    # regular python
+    class _PyObject(ctypes.Structure):
+        pass
+    _PyObject._fields_ = [
+        ('ob_refcnt', _Py_ssize_t),
+        ('ob_type', ctypes.POINTER(_PyObject))
+    ]
+
+    # python with trace
+    if hasattr(sys, 'getobjects'):
+        class _PyObject(ctypes.Structure):
+            pass
+        _PyObject._fields_ = [
+            ('_ob_next', ctypes.POINTER(_PyObject)),
+            ('_ob_prev', ctypes.POINTER(_PyObject)),
+            ('ob_refcnt', _Py_ssize_t),
+            ('ob_type', ctypes.POINTER(_PyObject))
+        ]
+
+    class _Traceback(_PyObject):
+        pass
+    _Traceback._fields_ = [
+        ('tb_next', ctypes.POINTER(_Traceback)),
+        ('tb_frame', ctypes.POINTER(_PyObject)),
+        ('tb_lasti', ctypes.c_int),
+        ('tb_lineno', ctypes.c_int)
+    ]
+
+    def tb_set_next(tb, next):
+        """Set the tb_next attribute of a traceback object."""
+        if not (isinstance(tb, TracebackType) and
+                (next is None or isinstance(next, TracebackType))):
+            raise TypeError('tb_set_next arguments must be traceback objects')
+        obj = _Traceback.from_address(id(tb))
+        if tb.tb_next is not None:
+            old = _Traceback.from_address(id(tb.tb_next))
+            old.ob_refcnt -= 1
+        if next is None:
+            obj.tb_next = ctypes.POINTER(_Traceback)()
+        else:
+            next = _Traceback.from_address(id(next))
+            next.ob_refcnt += 1
+            obj.tb_next = ctypes.pointer(next)
+
+    return tb_set_next
+
+
+# try to get a tb_set_next implementation if we don't have transparent
+# proxies.
+tb_set_next = None
+if tproxy is None:
+    try:
+        tb_set_next = _init_ugly_crap()
+    except:
+        pass
+    del _init_ugly_crap

python/jinja2/defaults.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.defaults
+    ~~~~~~~~~~~~~~~
+
+    Jinja default filters and tags.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+from jinja2._compat import range_type
+from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace
+
+
+# defaults for the parser / lexer
+BLOCK_START_STRING = '{%'
+BLOCK_END_STRING = '%}'
+VARIABLE_START_STRING = '{{'
+VARIABLE_END_STRING = '}}'
+COMMENT_START_STRING = '{#'
+COMMENT_END_STRING = '#}'
+LINE_STATEMENT_PREFIX = None
+LINE_COMMENT_PREFIX = None
+TRIM_BLOCKS = False
+LSTRIP_BLOCKS = False
+NEWLINE_SEQUENCE = '\n'
+KEEP_TRAILING_NEWLINE = False
+
+
+# default filters, tests and namespace
+from jinja2.filters import FILTERS as DEFAULT_FILTERS
+from jinja2.tests import TESTS as DEFAULT_TESTS
+DEFAULT_NAMESPACE = {
+    'range':        range_type,
+    'dict':         dict,
+    'lipsum':       generate_lorem_ipsum,
+    'cycler':       Cycler,
+    'joiner':       Joiner,
+    'namespace':    Namespace
+}
+
+
+# default policies
+DEFAULT_POLICIES = {
+    'compiler.ascii_str':   True,
+    'urlize.rel':           'noopener',
+    'urlize.target':        None,
+    'truncate.leeway':      5,
+    'json.dumps_function':  None,
+    'json.dumps_kwargs':    {'sort_keys': True},
+    'ext.i18n.trimmed':     False,
+}
+
+
+# export all constants
+__all__ = tuple(x for x in locals().keys() if x.isupper())

python/jinja2/exceptions.py

@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.exceptions
+    ~~~~~~~~~~~~~~~~~
+
+    Jinja exceptions.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+from jinja2._compat import imap, text_type, PY2, implements_to_string
+
+
+class TemplateError(Exception):
+    """Baseclass for all template errors."""
+
+    if PY2:
+        def __init__(self, message=None):
+            if message is not None:
+                message = text_type(message).encode('utf-8')
+            Exception.__init__(self, message)
+
+        @property
+        def message(self):
+            if self.args:
+                message = self.args[0]
+                if message is not None:
+                    return message.decode('utf-8', 'replace')
+
+        def __unicode__(self):
+            return self.message or u''
+    else:
+        def __init__(self, message=None):
+            Exception.__init__(self, message)
+
+        @property
+        def message(self):
+            if self.args:
+                message = self.args[0]
+                if message is not None:
+                    return message
+
+
+@implements_to_string
+class TemplateNotFound(IOError, LookupError, TemplateError):
+    """Raised if a template does not exist."""
+
+    # looks weird, but removes the warning descriptor that just
+    # bogusly warns us about message being deprecated
+    message = None
+
+    def __init__(self, name, message=None):
+        IOError.__init__(self)
+        if message is None:
+            message = name
+        self.message = message
+        self.name = name
+        self.templates = [name]
+
+    def __str__(self):
+        return self.message
+
+
+class TemplatesNotFound(TemplateNotFound):
+    """Like :class:`TemplateNotFound` but raised if multiple templates
+    are selected.  This is a subclass of :class:`TemplateNotFound`
+    exception, so just catching the base exception will catch both.
+
+    .. versionadded:: 2.2
+    """
+
+    def __init__(self, names=(), message=None):
+        if message is None:
+            message = u'none of the templates given were found: ' + \
+                      u', '.join(imap(text_type, names))
+        TemplateNotFound.__init__(self, names and names[-1] or None, message)
+        self.templates = list(names)
+
+
+@implements_to_string
+class TemplateSyntaxError(TemplateError):
+    """Raised to tell the user that there is a problem with the template."""
+
+    def __init__(self, message, lineno, name=None, filename=None):
+        TemplateError.__init__(self, message)
+        self.lineno = lineno
+        self.name = name
+        self.filename = filename
+        self.source = None
+
+        # this is set to True if the debug.translate_syntax_error
+        # function translated the syntax error into a new traceback
+        self.translated = False
+
+    def __str__(self):
+        # for translated errors we only return the message
+        if self.translated:
+            return self.message
+
+        # otherwise attach some stuff
+        location = 'line %d' % self.lineno
+        name = self.filename or self.name
+        if name:
+            location = 'File "%s", %s' % (name, location)
+        lines = [self.message, '  ' + location]
+
+        # if the source is set, add the line to the output
+        if self.source is not None:
+            try:
+                line = self.source.splitlines()[self.lineno - 1]
+            except IndexError:
+                line = None
+            if line:
+                lines.append('    ' + line.strip())
+
+        return u'\n'.join(lines)
+
+
+class TemplateAssertionError(TemplateSyntaxError):
+    """Like a template syntax error, but covers cases where something in the
+    template caused an error at compile time that wasn't necessarily caused
+    by a syntax error.  However it's a direct subclass of
+    :exc:`TemplateSyntaxError` and has the same attributes.
+    """
+
+
+class TemplateRuntimeError(TemplateError):
+    """A generic runtime error in the template engine.  Under some situations
+    Jinja may raise this exception.
+    """
+
+
+class UndefinedError(TemplateRuntimeError):
+    """Raised if a template tries to operate on :class:`Undefined`."""
+
+
+class SecurityError(TemplateRuntimeError):
+    """Raised if a template tries to do something insecure if the
+    sandbox is enabled.
+    """
+
+
+class FilterArgumentError(TemplateRuntimeError):
+    """This error is raised if a filter was called with inappropriate
+    arguments
+    """

python/jinja2/ext.py

@@ -0,0 +1,627 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.ext
+    ~~~~~~~~~~
+
+    Jinja extensions allow to add custom tags similar to the way django custom
+    tags work.  By default two example extensions exist: an i18n and a cache
+    extension.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+import re
+
+from jinja2 import nodes
+from jinja2.defaults import BLOCK_START_STRING, \
+     BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
+     COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
+     LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
+     KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
+from jinja2.environment import Environment
+from jinja2.runtime import concat
+from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
+from jinja2.utils import contextfunction, import_string, Markup
+from jinja2._compat import with_metaclass, string_types, iteritems
+
+
+# the only real useful gettext functions for a Jinja template.  Note
+# that ugettext must be assigned to gettext as Jinja doesn't support
+# non unicode strings.
+GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
+
+
+class ExtensionRegistry(type):
+    """Gives the extension an unique identifier."""
+
+    def __new__(cls, name, bases, d):
+        rv = type.__new__(cls, name, bases, d)
+        rv.identifier = rv.__module__ + '.' + rv.__name__
+        return rv
+
+
+class Extension(with_metaclass(ExtensionRegistry, object)):
+    """Extensions can be used to add extra functionality to the Jinja template
+    system at the parser level.  Custom extensions are bound to an environment
+    but may not store environment specific data on `self`.  The reason for
+    this is that an extension can be bound to another environment (for
+    overlays) by creating a copy and reassigning the `environment` attribute.
+
+    As extensions are created by the environment they cannot accept any
+    arguments for configuration.  One may want to work around that by using
+    a factory function, but that is not possible as extensions are identified
+    by their import name.  The correct way to configure the extension is
+    storing the configuration values on the environment.  Because this way the
+    environment ends up acting as central configuration storage the
+    attributes may clash which is why extensions have to ensure that the names
+    they choose for configuration are not too generic.  ``prefix`` for example
+    is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
+    name as includes the name of the extension (fragment cache).
+    """
+
+    #: if this extension parses this is the list of tags it's listening to.
+    tags = set()
+
+    #: the priority of that extension.  This is especially useful for
+    #: extensions that preprocess values.  A lower value means higher
+    #: priority.
+    #:
+    #: .. versionadded:: 2.4
+    priority = 100
+
+    def __init__(self, environment):
+        self.environment = environment
+
+    def bind(self, environment):
+        """Create a copy of this extension bound to another environment."""
+        rv = object.__new__(self.__class__)
+        rv.__dict__.update(self.__dict__)
+        rv.environment = environment
+        return rv
+
+    def preprocess(self, source, name, filename=None):
+        """This method is called before the actual lexing and can be used to
+        preprocess the source.  The `filename` is optional.  The return value
+        must be the preprocessed source.
+        """
+        return source
+
+    def filter_stream(self, stream):
+        """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
+        to filter tokens returned.  This method has to return an iterable of
+        :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
+        :class:`~jinja2.lexer.TokenStream`.
+
+        In the `ext` folder of the Jinja2 source distribution there is a file
+        called `inlinegettext.py` which implements a filter that utilizes this
+        method.
+        """
+        return stream
+
+    def parse(self, parser):
+        """If any of the :attr:`tags` matched this method is called with the
+        parser as first argument.  The token the parser stream is pointing at
+        is the name token that matched.  This method has to return one or a
+        list of multiple nodes.
+        """
+        raise NotImplementedError()
+
+    def attr(self, name, lineno=None):
+        """Return an attribute node for the current extension.  This is useful
+        to pass constants on extensions to generated template code.
+
+        ::
+
+            self.attr('_my_attribute', lineno=lineno)
+        """
+        return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
+
+    def call_method(self, name, args=None, kwargs=None, dyn_args=None,
+                    dyn_kwargs=None, lineno=None):
+        """Call a method of the extension.  This is a shortcut for
+        :meth:`attr` + :class:`jinja2.nodes.Call`.
+        """
+        if args is None:
+            args = []
+        if kwargs is None:
+            kwargs = []
+        return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
+                          dyn_args, dyn_kwargs, lineno=lineno)
+
+
+@contextfunction
+def _gettext_alias(__context, *args, **kwargs):
+    return __context.call(__context.resolve('gettext'), *args, **kwargs)
+
+
+def _make_new_gettext(func):
+    @contextfunction
+    def gettext(__context, __string, **variables):
+        rv = __context.call(func, __string)
+        if __context.eval_ctx.autoescape:
+            rv = Markup(rv)
+        return rv % variables
+    return gettext
+
+
+def _make_new_ngettext(func):
+    @contextfunction
+    def ngettext(__context, __singular, __plural, __num, **variables):
+        variables.setdefault('num', __num)
+        rv = __context.call(func, __singular, __plural, __num)
+        if __context.eval_ctx.autoescape:
+            rv = Markup(rv)
+        return rv % variables
+    return ngettext
+
+
+class InternationalizationExtension(Extension):
+    """This extension adds gettext support to Jinja2."""
+    tags = set(['trans'])
+
+    # TODO: the i18n extension is currently reevaluating values in a few
+    # situations.  Take this example:
+    #   {% trans count=something() %}{{ count }} foo{% pluralize
+    #     %}{{ count }} fooss{% endtrans %}
+    # something is called twice here.  One time for the gettext value and
+    # the other time for the n-parameter of the ngettext function.
+
+    def __init__(self, environment):
+        Extension.__init__(self, environment)
+        environment.globals['_'] = _gettext_alias
+        environment.extend(
+            install_gettext_translations=self._install,
+            install_null_translations=self._install_null,
+            install_gettext_callables=self._install_callables,
+            uninstall_gettext_translations=self._uninstall,
+            extract_translations=self._extract,
+            newstyle_gettext=False
+        )
+
+    def _install(self, translations, newstyle=None):
+        gettext = getattr(translations, 'ugettext', None)
+        if gettext is None:
+            gettext = translations.gettext
+        ngettext = getattr(translations, 'ungettext', None)
+        if ngettext is None:
+            ngettext = translations.ngettext
+        self._install_callables(gettext, ngettext, newstyle)
+
+    def _install_null(self, newstyle=None):
+        self._install_callables(
+            lambda x: x,
+            lambda s, p, n: (n != 1 and (p,) or (s,))[0],
+            newstyle
+        )
+
+    def _install_callables(self, gettext, ngettext, newstyle=None):
+        if newstyle is not None:
+            self.environment.newstyle_gettext = newstyle
+        if self.environment.newstyle_gettext:
+            gettext = _make_new_gettext(gettext)
+            ngettext = _make_new_ngettext(ngettext)
+        self.environment.globals.update(
+            gettext=gettext,
+            ngettext=ngettext
+        )
+
+    def _uninstall(self, translations):
+        for key in 'gettext', 'ngettext':
+            self.environment.globals.pop(key, None)
+
+    def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
+        if isinstance(source, string_types):
+            source = self.environment.parse(source)
+        return extract_from_ast(source, gettext_functions)
+
+    def parse(self, parser):
+        """Parse a translatable tag."""
+        lineno = next(parser.stream).lineno
+        num_called_num = False
+
+        # find all the variables referenced.  Additionally a variable can be
+        # defined in the body of the trans block too, but this is checked at
+        # a later state.
+        plural_expr = None
+        plural_expr_assignment = None
+        variables = {}
+        trimmed = None
+        while parser.stream.current.type != 'block_end':
+            if variables:
+                parser.stream.expect('comma')
+
+            # skip colon for python compatibility
+            if parser.stream.skip_if('colon'):
+                break
+
+            name = parser.stream.expect('name')
+            if name.value in variables:
+                parser.fail('translatable variable %r defined twice.' %
+                            name.value, name.lineno,
+                            exc=TemplateAssertionError)
+
+            # expressions
+            if parser.stream.current.type == 'assign':
+                next(parser.stream)
+                variables[name.value] = var = parser.parse_expression()
+            elif trimmed is None and name.value in ('trimmed', 'notrimmed'):
+                trimmed = name.value == 'trimmed'
+                continue
+            else:
+                variables[name.value] = var = nodes.Name(name.value, 'load')
+
+            if plural_expr is None:
+                if isinstance(var, nodes.Call):
+                    plural_expr = nodes.Name('_trans', 'load')
+                    variables[name.value] = plural_expr
+                    plural_expr_assignment = nodes.Assign(
+                        nodes.Name('_trans', 'store'), var)
+                else:
+                    plural_expr = var
+                num_called_num = name.value == 'num'
+
+        parser.stream.expect('block_end')
+
+        plural = None
+        have_plural = False
+        referenced = set()
+
+        # now parse until endtrans or pluralize
+        singular_names, singular = self._parse_block(parser, True)
+        if singular_names:
+            referenced.update(singular_names)
+            if plural_expr is None:
+                plural_expr = nodes.Name(singular_names[0], 'load')
+                num_called_num = singular_names[0] == 'num'
+
+        # if we have a pluralize block, we parse that too
+        if parser.stream.current.test('name:pluralize'):
+            have_plural = True
+            next(parser.stream)
+            if parser.stream.current.type != 'block_end':
+                name = parser.stream.expect('name')
+                if name.value not in variables:
+                    parser.fail('unknown variable %r for pluralization' %
+                                name.value, name.lineno,
+                                exc=TemplateAssertionError)
+                plural_expr = variables[name.value]
+                num_called_num = name.value == 'num'
+            parser.stream.expect('block_end')
+            plural_names, plural = self._parse_block(parser, False)
+            next(parser.stream)
+            referenced.update(plural_names)
+        else:
+            next(parser.stream)
+
+        # register free names as simple name expressions
+        for var in referenced:
+            if var not in variables:
+                variables[var] = nodes.Name(var, 'load')
+
+        if not have_plural:
+            plural_expr = None
+        elif plural_expr is None:
+            parser.fail('pluralize without variables', lineno)
+
+        if trimmed is None:
+            trimmed = self.environment.policies['ext.i18n.trimmed']
+        if trimmed:
+            singular = self._trim_whitespace(singular)
+            if plural:
+                plural = self._trim_whitespace(plural)
+
+        node = self._make_node(singular, plural, variables, plural_expr,
+                               bool(referenced),
+                               num_called_num and have_plural)
+        node.set_lineno(lineno)
+        if plural_expr_assignment is not None:
+            return [plural_expr_assignment, node]
+        else:
+            return node
+
+    def _trim_whitespace(self, string, _ws_re=re.compile(r'\s*\n\s*')):
+        return _ws_re.sub(' ', string.strip())
+
+    def _parse_block(self, parser, allow_pluralize):
+        """Parse until the next block tag with a given name."""
+        referenced = []
+        buf = []
+        while 1:
+            if parser.stream.current.type == 'data':
+                buf.append(parser.stream.current.value.replace('%', '%%'))
+                next(parser.stream)
+            elif parser.stream.current.type == 'variable_begin':
+                next(parser.stream)
+                name = parser.stream.expect('name').value
+                referenced.append(name)
+                buf.append('%%(%s)s' % name)
+                parser.stream.expect('variable_end')
+            elif parser.stream.current.type == 'block_begin':
+                next(parser.stream)
+                if parser.stream.current.test('name:endtrans'):
+                    break
+                elif parser.stream.current.test('name:pluralize'):
+                    if allow_pluralize:
+                        break
+                    parser.fail('a translatable section can have only one '
+                                'pluralize section')
+                parser.fail('control structures in translatable sections are '
+                            'not allowed')
+            elif parser.stream.eos:
+                parser.fail('unclosed translation block')
+            else:
+                assert False, 'internal parser error'
+
+        return referenced, concat(buf)
+
+    def _make_node(self, singular, plural, variables, plural_expr,
+                   vars_referenced, num_called_num):
+        """Generates a useful node from the data provided."""
+        # no variables referenced?  no need to escape for old style
+        # gettext invocations only if there are vars.
+        if not vars_referenced and not self.environment.newstyle_gettext:
+            singular = singular.replace('%%', '%')
+            if plural:
+                plural = plural.replace('%%', '%')
+
+        # singular only:
+        if plural_expr is None:
+            gettext = nodes.Name('gettext', 'load')
+            node = nodes.Call(gettext, [nodes.Const(singular)],
+                              [], None, None)
+
+        # singular and plural
+        else:
+            ngettext = nodes.Name('ngettext', 'load')
+            node = nodes.Call(ngettext, [
+                nodes.Const(singular),
+                nodes.Const(plural),
+                plural_expr
+            ], [], None, None)
+
+        # in case newstyle gettext is used, the method is powerful
+        # enough to handle the variable expansion and autoescape
+        # handling itself
+        if self.environment.newstyle_gettext:
+            for key, value in iteritems(variables):
+                # the function adds that later anyways in case num was
+                # called num, so just skip it.
+                if num_called_num and key == 'num':
+                    continue
+                node.kwargs.append(nodes.Keyword(key, value))
+
+        # otherwise do that here
+        else:
+            # mark the return value as safe if we are in an
+            # environment with autoescaping turned on
+            node = nodes.MarkSafeIfAutoescape(node)
+            if variables:
+                node = nodes.Mod(node, nodes.Dict([
+                    nodes.Pair(nodes.Const(key), value)
+                    for key, value in variables.items()
+                ]))
+        return nodes.Output([node])
+
+
+class ExprStmtExtension(Extension):
+    """Adds a `do` tag to Jinja2 that works like the print statement just
+    that it doesn't print the return value.
+    """
+    tags = set(['do'])
+
+    def parse(self, parser):
+        node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
+        node.node = parser.parse_tuple()
+        return node
+
+
+class LoopControlExtension(Extension):
+    """Adds break and continue to the template engine."""
+    tags = set(['break', 'continue'])
+
+    def parse(self, parser):
+        token = next(parser.stream)
+        if token.value == 'break':
+            return nodes.Break(lineno=token.lineno)
+        return nodes.Continue(lineno=token.lineno)
+
+
+class WithExtension(Extension):
+    pass
+
+
+class AutoEscapeExtension(Extension):
+    pass
+
+
+def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
+                     babel_style=True):
+    """Extract localizable strings from the given template node.  Per
+    default this function returns matches in babel style that means non string
+    parameters as well as keyword arguments are returned as `None`.  This
+    allows Babel to figure out what you really meant if you are using
+    gettext functions that allow keyword arguments for placeholder expansion.
+    If you don't want that behavior set the `babel_style` parameter to `False`
+    which causes only strings to be returned and parameters are always stored
+    in tuples.  As a consequence invalid gettext calls (calls without a single
+    string parameter or string parameters after non-string parameters) are
+    skipped.
+
+    This example explains the behavior:
+
+    >>> from jinja2 import Environment
+    >>> env = Environment()
+    >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
+    >>> list(extract_from_ast(node))
+    [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
+    >>> list(extract_from_ast(node, babel_style=False))
+    [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
+
+    For every string found this function yields a ``(lineno, function,
+    message)`` tuple, where:
+
+    * ``lineno`` is the number of the line on which the string was found,
+    * ``function`` is the name of the ``gettext`` function used (if the
+      string was extracted from embedded Python code), and
+    *  ``message`` is the string itself (a ``unicode`` object, or a tuple
+       of ``unicode`` objects for functions with multiple string arguments).
+
+    This extraction function operates on the AST and is because of that unable
+    to extract any comments.  For comment support you have to use the babel
+    extraction interface or extract comments yourself.
+    """
+    for node in node.find_all(nodes.Call):
+        if not isinstance(node.node, nodes.Name) or \
+           node.node.name not in gettext_functions:
+            continue
+
+        strings = []
+        for arg in node.args:
+            if isinstance(arg, nodes.Const) and \
+               isinstance(arg.value, string_types):
+                strings.append(arg.value)
+            else:
+                strings.append(None)
+
+        for arg in node.kwargs:
+            strings.append(None)
+        if node.dyn_args is not None:
+            strings.append(None)
+        if node.dyn_kwargs is not None:
+            strings.append(None)
+
+        if not babel_style:
+            strings = tuple(x for x in strings if x is not None)
+            if not strings:
+                continue
+        else:
+            if len(strings) == 1:
+                strings = strings[0]
+            else:
+                strings = tuple(strings)
+        yield node.lineno, node.node.name, strings
+
+
+class _CommentFinder(object):
+    """Helper class to find comments in a token stream.  Can only
+    find comments for gettext calls forwards.  Once the comment
+    from line 4 is found, a comment for line 1 will not return a
+    usable value.
+    """
+
+    def __init__(self, tokens, comment_tags):
+        self.tokens = tokens
+        self.comment_tags = comment_tags
+        self.offset = 0
+        self.last_lineno = 0
+
+    def find_backwards(self, offset):
+        try:
+            for _, token_type, token_value in \
+                    reversed(self.tokens[self.offset:offset]):
+                if token_type in ('comment', 'linecomment'):
+                    try:
+                        prefix, comment = token_value.split(None, 1)
+                    except ValueError:
+                        continue
+                    if prefix in self.comment_tags:
+                        return [comment.rstrip()]
+            return []
+        finally:
+            self.offset = offset
+
+    def find_comments(self, lineno):
+        if not self.comment_tags or self.last_lineno > lineno:
+            return []
+        for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
+            if token_lineno > lineno:
+                return self.find_backwards(self.offset + idx)
+        return self.find_backwards(len(self.tokens))
+
+
+def babel_extract(fileobj, keywords, comment_tags, options):
+    """Babel extraction method for Jinja templates.
+
+    .. versionchanged:: 2.3
+       Basic support for translation comments was added.  If `comment_tags`
+       is now set to a list of keywords for extraction, the extractor will
+       try to find the best preceeding comment that begins with one of the
+       keywords.  For best results, make sure to not have more than one
+       gettext call in one line of code and the matching comment in the
+       same line or the line before.
+
+    .. versionchanged:: 2.5.1
+       The `newstyle_gettext` flag can be set to `True` to enable newstyle
+       gettext calls.
+
+    .. versionchanged:: 2.7
+       A `silent` option can now be provided.  If set to `False` template
+       syntax errors are propagated instead of being ignored.
+
+    :param fileobj: the file-like object the messages should be extracted from
+    :param keywords: a list of keywords (i.e. function names) that should be
+                     recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and include
+                         in the results.
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
+             (comments will be empty currently)
+    """
+    extensions = set()
+    for extension in options.get('extensions', '').split(','):
+        extension = extension.strip()
+        if not extension:
+            continue
+        extensions.add(import_string(extension))
+    if InternationalizationExtension not in extensions:
+        extensions.add(InternationalizationExtension)
+
+    def getbool(options, key, default=False):
+        return options.get(key, str(default)).lower() in \
+            ('1', 'on', 'yes', 'true')
+
+    silent = getbool(options, 'silent', True)
+    environment = Environment(
+        options.get('block_start_string', BLOCK_START_STRING),
+        options.get('block_end_string', BLOCK_END_STRING),
+        options.get('variable_start_string', VARIABLE_START_STRING),
+        options.get('variable_end_string', VARIABLE_END_STRING),
+        options.get('comment_start_string', COMMENT_START_STRING),
+        options.get('comment_end_string', COMMENT_END_STRING),
+        options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
+        options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
+        getbool(options, 'trim_blocks', TRIM_BLOCKS),
+        getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS),
+        NEWLINE_SEQUENCE,
+        getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE),
+        frozenset(extensions),
+        cache_size=0,
+        auto_reload=False
+    )
+
+    if getbool(options, 'trimmed'):
+        environment.policies['ext.i18n.trimmed'] = True
+    if getbool(options, 'newstyle_gettext'):
+        environment.newstyle_gettext = True
+
+    source = fileobj.read().decode(options.get('encoding', 'utf-8'))
+    try:
+        node = environment.parse(source)
+        tokens = list(environment.lex(environment.preprocess(source)))
+    except TemplateSyntaxError as e:
+        if not silent:
+            raise
+        # skip templates with syntax errors
+        return
+
+    finder = _CommentFinder(tokens, comment_tags)
+    for lineno, func, message in extract_from_ast(node, keywords):
+        yield lineno, func, message, finder.find_comments(lineno)
+
+
+#: nicer import names
+i18n = InternationalizationExtension
+do = ExprStmtExtension
+loopcontrols = LoopControlExtension
+with_ = WithExtension
+autoescape = AutoEscapeExtension

python/jinja2/idtracking.py

@@ -0,0 +1,286 @@
+from jinja2.visitor import NodeVisitor
+from jinja2._compat import iteritems
+
+
+VAR_LOAD_PARAMETER = 'param'
+VAR_LOAD_RESOLVE = 'resolve'
+VAR_LOAD_ALIAS = 'alias'
+VAR_LOAD_UNDEFINED = 'undefined'
+
+
+def find_symbols(nodes, parent_symbols=None):
+    sym = Symbols(parent=parent_symbols)
+    visitor = FrameSymbolVisitor(sym)
+    for node in nodes:
+        visitor.visit(node)
+    return sym
+
+
+def symbols_for_node(node, parent_symbols=None):
+    sym = Symbols(parent=parent_symbols)
+    sym.analyze_node(node)
+    return sym
+
+
+class Symbols(object):
+
+    def __init__(self, parent=None, level=None):
+        if level is None:
+            if parent is None:
+                level = 0
+            else:
+                level = parent.level + 1
+        self.level = level
+        self.parent = parent
+        self.refs = {}
+        self.loads = {}
+        self.stores = set()
+
+    def analyze_node(self, node, **kwargs):
+        visitor = RootVisitor(self)
+        visitor.visit(node, **kwargs)
+
+    def _define_ref(self, name, load=None):
+        ident = 'l_%d_%s' % (self.level, name)
+        self.refs[name] = ident
+        if load is not None:
+            self.loads[ident] = load
+        return ident
+
+    def find_load(self, target):
+        if target in self.loads:
+            return self.loads[target]
+        if self.parent is not None:
+            return self.parent.find_load(target)
+
+    def find_ref(self, name):
+        if name in self.refs:
+            return self.refs[name]
+        if self.parent is not None:
+            return self.parent.find_ref(name)
+
+    def ref(self, name):
+        rv = self.find_ref(name)
+        if rv is None:
+            raise AssertionError('Tried to resolve a name to a reference that '
+                                 'was unknown to the frame (%r)' % name)
+        return rv
+
+    def copy(self):
+        rv = object.__new__(self.__class__)
+        rv.__dict__.update(self.__dict__)
+        rv.refs = self.refs.copy()
+        rv.loads = self.loads.copy()
+        rv.stores = self.stores.copy()
+        return rv
+
+    def store(self, name):
+        self.stores.add(name)
+
+        # If we have not see the name referenced yet, we need to figure
+        # out what to set it to.
+        if name not in self.refs:
+            # If there is a parent scope we check if the name has a
+            # reference there.  If it does it means we might have to alias
+            # to a variable there.
+            if self.parent is not None:
+                outer_ref = self.parent.find_ref(name)
+                if outer_ref is not None:
+                    self._define_ref(name, load=(VAR_LOAD_ALIAS, outer_ref))
+                    return
+
+            # Otherwise we can just set it to undefined.
+            self._define_ref(name, load=(VAR_LOAD_UNDEFINED, None))
+
+    def declare_parameter(self, name):
+        self.stores.add(name)
+        return self._define_ref(name, load=(VAR_LOAD_PARAMETER, None))
+
+    def load(self, name):
+        target = self.find_ref(name)
+        if target is None:
+            self._define_ref(name, load=(VAR_LOAD_RESOLVE, name))
+
+    def branch_update(self, branch_symbols):
+        stores = {}
+        for branch in branch_symbols:
+            for target in branch.stores:
+                if target in self.stores:
+                    continue
+                stores[target] = stores.get(target, 0) + 1
+
+        for sym in branch_symbols:
+            self.refs.update(sym.refs)
+            self.loads.update(sym.loads)
+            self.stores.update(sym.stores)
+
+        for name, branch_count in iteritems(stores):
+            if branch_count == len(branch_symbols):
+                continue
+            target = self.find_ref(name)
+            assert target is not None, 'should not happen'
+
+            if self.parent is not None:
+                outer_target = self.parent.find_ref(name)
+                if outer_target is not None:
+                    self.loads[target] = (VAR_LOAD_ALIAS, outer_target)
+                    continue
+            self.loads[target] = (VAR_LOAD_RESOLVE, name)
+
+    def dump_stores(self):
+        rv = {}
+        node = self
+        while node is not None:
+            for name in node.stores:
+                if name not in rv:
+                    rv[name] = self.find_ref(name)
+            node = node.parent
+        return rv
+
+    def dump_param_targets(self):
+        rv = set()
+        node = self
+        while node is not None:
+            for target, (instr, _) in iteritems(self.loads):
+                if instr == VAR_LOAD_PARAMETER:
+                    rv.add(target)
+            node = node.parent
+        return rv
+
+
+class RootVisitor(NodeVisitor):
+
+    def __init__(self, symbols):
+        self.sym_visitor = FrameSymbolVisitor(symbols)
+
+    def _simple_visit(self, node, **kwargs):
+        for child in node.iter_child_nodes():
+            self.sym_visitor.visit(child)
+
+    visit_Template = visit_Block = visit_Macro = visit_FilterBlock = \
+        visit_Scope = visit_If = visit_ScopedEvalContextModifier = \
+        _simple_visit
+
+    def visit_AssignBlock(self, node, **kwargs):
+        for child in node.body:
+            self.sym_visitor.visit(child)
+
+    def visit_CallBlock(self, node, **kwargs):
+        for child in node.iter_child_nodes(exclude=('call',)):
+            self.sym_visitor.visit(child)
+
+    def visit_OverlayScope(self, node, **kwargs):
+        for child in node.body:
+            self.sym_visitor.visit(child)
+
+    def visit_For(self, node, for_branch='body', **kwargs):
+        if for_branch == 'body':
+            self.sym_visitor.visit(node.target, store_as_param=True)
+            branch = node.body
+        elif for_branch == 'else':
+            branch = node.else_
+        elif for_branch == 'test':
+            self.sym_visitor.visit(node.target, store_as_param=True)
+            if node.test is not None:
+                self.sym_visitor.visit(node.test)
+            return
+        else:
+            raise RuntimeError('Unknown for branch')
+        for item in branch or ():
+            self.sym_visitor.visit(item)
+
+    def visit_With(self, node, **kwargs):
+        for target in node.targets:
+            self.sym_visitor.visit(target)
+        for child in node.body:
+            self.sym_visitor.visit(child)
+
+    def generic_visit(self, node, *args, **kwargs):
+        raise NotImplementedError('Cannot find symbols for %r' %
+                                  node.__class__.__name__)
+
+
+class FrameSymbolVisitor(NodeVisitor):
+    """A visitor for `Frame.inspect`."""
+
+    def __init__(self, symbols):
+        self.symbols = symbols
+
+    def visit_Name(self, node, store_as_param=False, **kwargs):
+        """All assignments to names go through this function."""
+        if store_as_param or node.ctx == 'param':
+            self.symbols.declare_parameter(node.name)
+        elif node.ctx == 'store':
+            self.symbols.store(node.name)
+        elif node.ctx == 'load':
+            self.symbols.load(node.name)
+
+    def visit_NSRef(self, node, **kwargs):
+        self.symbols.load(node.name)
+
+    def visit_If(self, node, **kwargs):
+        self.visit(node.test, **kwargs)
+
+        original_symbols = self.symbols
+
+        def inner_visit(nodes):
+            self.symbols = rv = original_symbols.copy()
+            for subnode in nodes:
+                self.visit(subnode, **kwargs)
+            self.symbols = original_symbols
+            return rv
+
+        body_symbols = inner_visit(node.body)
+        elif_symbols = inner_visit(node.elif_)
+        else_symbols = inner_visit(node.else_ or ())
+
+        self.symbols.branch_update([body_symbols, elif_symbols, else_symbols])
+
+    def visit_Macro(self, node, **kwargs):
+        self.symbols.store(node.name)
+
+    def visit_Import(self, node, **kwargs):
+        self.generic_visit(node, **kwargs)
+        self.symbols.store(node.target)
+
+    def visit_FromImport(self, node, **kwargs):
+        self.generic_visit(node, **kwargs)
+        for name in node.names:
+            if isinstance(name, tuple):
+                self.symbols.store(name[1])
+            else:
+                self.symbols.store(name)
+
+    def visit_Assign(self, node, **kwargs):
+        """Visit assignments in the correct order."""
+        self.visit(node.node, **kwargs)
+        self.visit(node.target, **kwargs)
+
+    def visit_For(self, node, **kwargs):
+        """Visiting stops at for blocks.  However the block sequence
+        is visited as part of the outer scope.
+        """
+        self.visit(node.iter, **kwargs)
+
+    def visit_CallBlock(self, node, **kwargs):
+        self.visit(node.call, **kwargs)
+
+    def visit_FilterBlock(self, node, **kwargs):
+        self.visit(node.filter, **kwargs)
+
+    def visit_With(self, node, **kwargs):
+        for target in node.values:
+            self.visit(target)
+
+    def visit_AssignBlock(self, node, **kwargs):
+        """Stop visiting at block assigns."""
+        self.visit(node.target, **kwargs)
+
+    def visit_Scope(self, node, **kwargs):
+        """Stop visiting at scopes."""
+
+    def visit_Block(self, node, **kwargs):
+        """Stop visiting at blocks."""
+
+    def visit_OverlayScope(self, node, **kwargs):
+        """Do not visit into overlay scopes."""

python/jinja2/lexer.py

@@ -0,0 +1,739 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.lexer
+    ~~~~~~~~~~~~
+
+    This module implements a Jinja / Python combination lexer. The
+    `Lexer` class provided by this module is used to do some preprocessing
+    for Jinja.
+
+    On the one hand it filters out invalid operators like the bitshift
+    operators we don't allow in templates. On the other hand it separates
+    template code and python code in expressions.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+from collections import deque
+from operator import itemgetter
+
+from jinja2._compat import implements_iterator, intern, iteritems, text_type
+from jinja2.exceptions import TemplateSyntaxError
+from jinja2.utils import LRUCache
+
+# cache for the lexers. Exists in order to be able to have multiple
+# environments with the same lexer
+_lexer_cache = LRUCache(50)
+
+# static regular expressions
+whitespace_re = re.compile(r'\s+', re.U)
+string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+                       r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
+integer_re = re.compile(r'\d+')
+
+try:
+    # check if this Python supports Unicode identifiers
+    compile('föö', '<unknown>', 'eval')
+except SyntaxError:
+    # no Unicode support, use ASCII identifiers
+    name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
+    check_ident = False
+else:
+    # Unicode support, build a pattern to match valid characters, and set flag
+    # to use str.isidentifier to validate during lexing
+    from jinja2 import _identifier
+    name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern))
+    check_ident = True
+    # remove the pattern from memory after building the regex
+    import sys
+    del sys.modules['jinja2._identifier']
+    import jinja2
+    del jinja2._identifier
+    del _identifier
+
+float_re = re.compile(r'(?<!\.)\d+\.\d+')
+newline_re = re.compile(r'(\r\n|\r|\n)')
+
+# internal the tokens and keep references to them
+TOKEN_ADD = intern('add')
+TOKEN_ASSIGN = intern('assign')
+TOKEN_COLON = intern('colon')
+TOKEN_COMMA = intern('comma')
+TOKEN_DIV = intern('div')
+TOKEN_DOT = intern('dot')
+TOKEN_EQ = intern('eq')
+TOKEN_FLOORDIV = intern('floordiv')
+TOKEN_GT = intern('gt')
+TOKEN_GTEQ = intern('gteq')
+TOKEN_LBRACE = intern('lbrace')
+TOKEN_LBRACKET = intern('lbracket')
+TOKEN_LPAREN = intern('lparen')
+TOKEN_LT = intern('lt')
+TOKEN_LTEQ = intern('lteq')
+TOKEN_MOD = intern('mod')
+TOKEN_MUL = intern('mul')
+TOKEN_NE = intern('ne')
+TOKEN_PIPE = intern('pipe')
+TOKEN_POW = intern('pow')
+TOKEN_RBRACE = intern('rbrace')
+TOKEN_RBRACKET = intern('rbracket')
+TOKEN_RPAREN = intern('rparen')
+TOKEN_SEMICOLON = intern('semicolon')
+TOKEN_SUB = intern('sub')
+TOKEN_TILDE = intern('tilde')
+TOKEN_WHITESPACE = intern('whitespace')
+TOKEN_FLOAT = intern('float')
+TOKEN_INTEGER = intern('integer')
+TOKEN_NAME = intern('name')
+TOKEN_STRING = intern('string')
+TOKEN_OPERATOR = intern('operator')
+TOKEN_BLOCK_BEGIN = intern('block_begin')
+TOKEN_BLOCK_END = intern('block_end')
+TOKEN_VARIABLE_BEGIN = intern('variable_begin')
+TOKEN_VARIABLE_END = intern('variable_end')
+TOKEN_RAW_BEGIN = intern('raw_begin')
+TOKEN_RAW_END = intern('raw_end')
+TOKEN_COMMENT_BEGIN = intern('comment_begin')
+TOKEN_COMMENT_END = intern('comment_end')
+TOKEN_COMMENT = intern('comment')
+TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
+TOKEN_LINESTATEMENT_END = intern('linestatement_end')
+TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
+TOKEN_LINECOMMENT_END = intern('linecomment_end')
+TOKEN_LINECOMMENT = intern('linecomment')
+TOKEN_DATA = intern('data')
+TOKEN_INITIAL = intern('initial')
+TOKEN_EOF = intern('eof')
+
+# bind operators to token types
+operators = {
+    '+':            TOKEN_ADD,
+    '-':            TOKEN_SUB,
+    '/':            TOKEN_DIV,
+    '//':           TOKEN_FLOORDIV,
+    '*':            TOKEN_MUL,
+    '%':            TOKEN_MOD,
+    '**':           TOKEN_POW,
+    '~':            TOKEN_TILDE,
+    '[':            TOKEN_LBRACKET,
+    ']':            TOKEN_RBRACKET,
+    '(':            TOKEN_LPAREN,
+    ')':            TOKEN_RPAREN,
+    '{':            TOKEN_LBRACE,
+    '}':            TOKEN_RBRACE,
+    '==':           TOKEN_EQ,
+    '!=':           TOKEN_NE,
+    '>':            TOKEN_GT,
+    '>=':           TOKEN_GTEQ,
+    '<':            TOKEN_LT,
+    '<=':           TOKEN_LTEQ,
+    '=':            TOKEN_ASSIGN,
+    '.':            TOKEN_DOT,
+    ':':            TOKEN_COLON,
+    '|':            TOKEN_PIPE,
+    ',':            TOKEN_COMMA,
+    ';':            TOKEN_SEMICOLON
+}
+
+reverse_operators = dict([(v, k) for k, v in iteritems(operators)])
+assert len(operators) == len(reverse_operators), 'operators dropped'
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+                         sorted(operators, key=lambda x: -len(x))))
+
+ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
+                            TOKEN_COMMENT_END, TOKEN_WHITESPACE,
+                            TOKEN_LINECOMMENT_BEGIN, TOKEN_LINECOMMENT_END,
+                            TOKEN_LINECOMMENT])
+ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
+                             TOKEN_COMMENT, TOKEN_LINECOMMENT])
+
+
+def _describe_token_type(token_type):
+    if token_type in reverse_operators:
+        return reverse_operators[token_type]
+    return {
+        TOKEN_COMMENT_BEGIN:        'begin of comment',
+        TOKEN_COMMENT_END:          'end of comment',
+        TOKEN_COMMENT:              'comment',
+        TOKEN_LINECOMMENT:          'comment',
+        TOKEN_BLOCK_BEGIN:          'begin of statement block',
+        TOKEN_BLOCK_END:            'end of statement block',
+        TOKEN_VARIABLE_BEGIN:       'begin of print statement',
+        TOKEN_VARIABLE_END:         'end of print statement',
+        TOKEN_LINESTATEMENT_BEGIN:  'begin of line statement',
+        TOKEN_LINESTATEMENT_END:    'end of line statement',
+        TOKEN_DATA:                 'template data / text',
+        TOKEN_EOF:                  'end of template'
+    }.get(token_type, token_type)
+
+
+def describe_token(token):
+    """Returns a description of the token."""
+    if token.type == 'name':
+        return token.value
+    return _describe_token_type(token.type)
+
+
+def describe_token_expr(expr):
+    """Like `describe_token` but for token expressions."""
+    if ':' in expr:
+        type, value = expr.split(':', 1)
+        if type == 'name':
+            return value
+    else:
+        type = expr
+    return _describe_token_type(type)
+
+
+def count_newlines(value):
+    """Count the number of newline characters in the string.  This is
+    useful for extensions that filter a stream.
+    """
+    return len(newline_re.findall(value))
+
+
+def compile_rules(environment):
+    """Compiles all the rules from the environment into a list of rules."""
+    e = re.escape
+    rules = [
+        (len(environment.comment_start_string), 'comment',
+         e(environment.comment_start_string)),
+        (len(environment.block_start_string), 'block',
+         e(environment.block_start_string)),
+        (len(environment.variable_start_string), 'variable',
+         e(environment.variable_start_string))
+    ]
+
+    if environment.line_statement_prefix is not None:
+        rules.append((len(environment.line_statement_prefix), 'linestatement',
+                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
+    if environment.line_comment_prefix is not None:
+        rules.append((len(environment.line_comment_prefix), 'linecomment',
+                      r'(?:^|(?<=\S))[^\S\r\n]*' +
+                      e(environment.line_comment_prefix)))
+
+    return [x[1:] for x in sorted(rules, reverse=True)]
+
+
+class Failure(object):
+    """Class that raises a `TemplateSyntaxError` if called.
+    Used by the `Lexer` to specify known errors.
+    """
+
+    def __init__(self, message, cls=TemplateSyntaxError):
+        self.message = message
+        self.error_class = cls
+
+    def __call__(self, lineno, filename):
+        raise self.error_class(self.message, lineno, filename)
+
+
+class Token(tuple):
+    """Token class."""
+    __slots__ = ()
+    lineno, type, value = (property(itemgetter(x)) for x in range(3))
+
+    def __new__(cls, lineno, type, value):
+        return tuple.__new__(cls, (lineno, intern(str(type)), value))
+
+    def __str__(self):
+        if self.type in reverse_operators:
+            return reverse_operators[self.type]
+        elif self.type == 'name':
+            return self.value
+        return self.type
+
+    def test(self, expr):
+        """Test a token against a token expression.  This can either be a
+        token type or ``'token_type:token_value'``.  This can only test
+        against string values and types.
+        """
+        # here we do a regular string equality check as test_any is usually
+        # passed an iterable of not interned strings.
+        if self.type == expr:
+            return True
+        elif ':' in expr:
+            return expr.split(':', 1) == [self.type, self.value]
+        return False
+
+    def test_any(self, *iterable):
+        """Test against multiple token expressions."""
+        for expr in iterable:
+            if self.test(expr):
+                return True
+        return False
+
+    def __repr__(self):
+        return 'Token(%r, %r, %r)' % (
+            self.lineno,
+            self.type,
+            self.value
+        )
+
+
+@implements_iterator
+class TokenStreamIterator(object):
+    """The iterator for tokenstreams.  Iterate over the stream
+    until the eof token is reached.
+    """
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        token = self.stream.current
+        if token.type is TOKEN_EOF:
+            self.stream.close()
+            raise StopIteration()
+        next(self.stream)
+        return token
+
+
+@implements_iterator
+class TokenStream(object):
+    """A token stream is an iterable that yields :class:`Token`\\s.  The
+    parser however does not iterate over it but calls :meth:`next` to go
+    one token ahead.  The current active token is stored as :attr:`current`.
+    """
+
+    def __init__(self, generator, name, filename):
+        self._iter = iter(generator)
+        self._pushed = deque()
+        self.name = name
+        self.filename = filename
+        self.closed = False
+        self.current = Token(1, TOKEN_INITIAL, '')
+        next(self)
+
+    def __iter__(self):
+        return TokenStreamIterator(self)
+
+    def __bool__(self):
+        return bool(self._pushed) or self.current.type is not TOKEN_EOF
+    __nonzero__ = __bool__  # py2
+
+    eos = property(lambda x: not x, doc="Are we at the end of the stream?")
+
+    def push(self, token):
+        """Push a token back to the stream."""
+        self._pushed.append(token)
+
+    def look(self):
+        """Look at the next token."""
+        old_token = next(self)
+        result = self.current
+        self.push(result)
+        self.current = old_token
+        return result
+
+    def skip(self, n=1):
+        """Got n tokens ahead."""
+        for x in range(n):
+            next(self)
+
+    def next_if(self, expr):
+        """Perform the token test and return the token if it matched.
+        Otherwise the return value is `None`.
+        """
+        if self.current.test(expr):
+            return next(self)
+
+    def skip_if(self, expr):
+        """Like :meth:`next_if` but only returns `True` or `False`."""
+        return self.next_if(expr) is not None
+
+    def __next__(self):
+        """Go one token ahead and return the old one.
+
+        Use the built-in :func:`next` instead of calling this directly.
+        """
+        rv = self.current
+        if self._pushed:
+            self.current = self._pushed.popleft()
+        elif self.current.type is not TOKEN_EOF:
+            try:
+                self.current = next(self._iter)
+            except StopIteration:
+                self.close()
+        return rv
+
+    def close(self):
+        """Close the stream."""
+        self.current = Token(self.current.lineno, TOKEN_EOF, '')
+        self._iter = None
+        self.closed = True
+
+    def expect(self, expr):
+        """Expect a given token type and return it.  This accepts the same
+        argument as :meth:`jinja2.lexer.Token.test`.
+        """
+        if not self.current.test(expr):
+            expr = describe_token_expr(expr)
+            if self.current.type is TOKEN_EOF:
+                raise TemplateSyntaxError('unexpected end of template, '
+                                          'expected %r.' % expr,
+                                          self.current.lineno,
+                                          self.name, self.filename)
+            raise TemplateSyntaxError("expected token %r, got %r" %
+                                      (expr, describe_token(self.current)),
+                                      self.current.lineno,
+                                      self.name, self.filename)
+        try:
+            return self.current
+        finally:
+            next(self)
+
+
+def get_lexer(environment):
+    """Return a lexer which is probably cached."""
+    key = (environment.block_start_string,
+           environment.block_end_string,
+           environment.variable_start_string,
+           environment.variable_end_string,
+           environment.comment_start_string,
+           environment.comment_end_string,
+           environment.line_statement_prefix,
+           environment.line_comment_prefix,
+           environment.trim_blocks,
+           environment.lstrip_blocks,
+           environment.newline_sequence,
+           environment.keep_trailing_newline)
+    lexer = _lexer_cache.get(key)
+    if lexer is None:
+        lexer = Lexer(environment)
+        _lexer_cache[key] = lexer
+    return lexer
+
+
+class Lexer(object):
+    """Class that implements a lexer for a given environment. Automatically
+    created by the environment class, usually you don't have to do that.
+
+    Note that the lexer is not automatically bound to an environment.
+    Multiple environments can share the same lexer.
+    """
+
+    def __init__(self, environment):
+        # shortcuts
+        c = lambda x: re.compile(x, re.M | re.S)
+        e = re.escape
+
+        # lexing rules for tags
+        tag_rules = [
+            (whitespace_re, TOKEN_WHITESPACE, None),
+            (float_re, TOKEN_FLOAT, None),
+            (integer_re, TOKEN_INTEGER, None),
+            (name_re, TOKEN_NAME, None),
+            (string_re, TOKEN_STRING, None),
+            (operator_re, TOKEN_OPERATOR, None)
+        ]
+
+        # assemble the root lexing rule. because "|" is ungreedy
+        # we have to sort by length so that the lexer continues working
+        # as expected when we have parsing rules like <% for block and
+        # <%= for variables. (if someone wants asp like syntax)
+        # variables are just part of the rules if variable processing
+        # is required.
+        root_tag_rules = compile_rules(environment)
+
+        # block suffix if trimming is enabled
+        block_suffix_re = environment.trim_blocks and '\\n?' or ''
+
+        # strip leading spaces if lstrip_blocks is enabled
+        prefix_re = {}
+        if environment.lstrip_blocks:
+            # use '{%+' to manually disable lstrip_blocks behavior
+            no_lstrip_re = e('+')
+            # detect overlap between block and variable or comment strings
+            block_diff = c(r'^%s(.*)' % e(environment.block_start_string))
+            # make sure we don't mistake a block for a variable or a comment
+            m = block_diff.match(environment.comment_start_string)
+            no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
+            m = block_diff.match(environment.variable_start_string)
+            no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
+
+            # detect overlap between comment and variable strings
+            comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string))
+            m = comment_diff.match(environment.variable_start_string)
+            no_variable_re = m and r'(?!%s)' % e(m.group(1)) or ''
+
+            lstrip_re = r'^[ \t]*'
+            block_prefix_re = r'%s%s(?!%s)|%s\+?' % (
+                    lstrip_re,
+                    e(environment.block_start_string),
+                    no_lstrip_re,
+                    e(environment.block_start_string),
+                    )
+            comment_prefix_re = r'%s%s%s|%s\+?' % (
+                    lstrip_re,
+                    e(environment.comment_start_string),
+                    no_variable_re,
+                    e(environment.comment_start_string),
+                    )
+            prefix_re['block'] = block_prefix_re
+            prefix_re['comment'] = comment_prefix_re
+        else:
+            block_prefix_re = '%s' % e(environment.block_start_string)
+
+        self.newline_sequence = environment.newline_sequence
+        self.keep_trailing_newline = environment.keep_trailing_newline
+
+        # global lexing rules
+        self.rules = {
+            'root': [
+                # directives
+                (c('(.*?)(?:%s)' % '|'.join(
+                    [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+                        e(environment.block_start_string),
+                        block_prefix_re,
+                        e(environment.block_end_string),
+                        e(environment.block_end_string)
+                    )] + [
+                        r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r))
+                        for n, r in root_tag_rules
+                    ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
+                # data
+                (c('.+'), TOKEN_DATA, None)
+            ],
+            # comments
+            TOKEN_COMMENT_BEGIN: [
+                (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
+                    e(environment.comment_end_string),
+                    e(environment.comment_end_string),
+                    block_suffix_re
+                )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
+                (c('(.)'), (Failure('Missing end of comment tag'),), None)
+            ],
+            # blocks
+            TOKEN_BLOCK_BEGIN: [
+                (c(r'(?:\-%s\s*|%s)%s' % (
+                    e(environment.block_end_string),
+                    e(environment.block_end_string),
+                    block_suffix_re
+                )), TOKEN_BLOCK_END, '#pop'),
+            ] + tag_rules,
+            # variables
+            TOKEN_VARIABLE_BEGIN: [
+                (c(r'\-%s\s*|%s' % (
+                    e(environment.variable_end_string),
+                    e(environment.variable_end_string)
+                )), TOKEN_VARIABLE_END, '#pop')
+            ] + tag_rules,
+            # raw block
+            TOKEN_RAW_BEGIN: [
+                (c(r'(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
+                    e(environment.block_start_string),
+                    block_prefix_re,
+                    e(environment.block_end_string),
+                    e(environment.block_end_string),
+                    block_suffix_re
+                )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
+                (c('(.)'), (Failure('Missing end of raw directive'),), None)
+            ],
+            # line statements
+            TOKEN_LINESTATEMENT_BEGIN: [
+                (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
+            ] + tag_rules,
+            # line comments
+            TOKEN_LINECOMMENT_BEGIN: [
+                (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
+                 TOKEN_LINECOMMENT_END), '#pop')
+            ]
+        }
+
+    def _normalize_newlines(self, value):
+        """Called for strings and template data to normalize it to unicode."""
+        return newline_re.sub(self.newline_sequence, value)
+
+    def tokenize(self, source, name=None, filename=None, state=None):
+        """Calls tokeniter + tokenize and wraps it in a token stream.
+        """
+        stream = self.tokeniter(source, name, filename, state)
+        return TokenStream(self.wrap(stream, name, filename), name, filename)
+
+    def wrap(self, stream, name=None, filename=None):
+        """This is called with the stream as returned by `tokenize` and wraps
+        every token in a :class:`Token` and converts the value.
+        """
+        for lineno, token, value in stream:
+            if token in ignored_tokens:
+                continue
+            elif token == 'linestatement_begin':
+                token = 'block_begin'
+            elif token == 'linestatement_end':
+                token = 'block_end'
+            # we are not interested in those tokens in the parser
+            elif token in ('raw_begin', 'raw_end'):
+                continue
+            elif token == 'data':
+                value = self._normalize_newlines(value)
+            elif token == 'keyword':
+                token = value
+            elif token == 'name':
+                value = str(value)
+                if check_ident and not value.isidentifier():
+                    raise TemplateSyntaxError(
+                        'Invalid character in identifier',
+                        lineno, name, filename)
+            elif token == 'string':
+                # try to unescape string
+                try:
+                    value = self._normalize_newlines(value[1:-1]) \
+                        .encode('ascii', 'backslashreplace') \
+                        .decode('unicode-escape')
+                except Exception as e:
+                    msg = str(e).split(':')[-1].strip()
+                    raise TemplateSyntaxError(msg, lineno, name, filename)
+            elif token == 'integer':
+                value = int(value)
+            elif token == 'float':
+                value = float(value)
+            elif token == 'operator':
+                token = operators[value]
+            yield Token(lineno, token, value)
+
+    def tokeniter(self, source, name, filename=None, state=None):
+        """This method tokenizes the text and returns the tokens in a
+        generator.  Use this method if you just want to tokenize a template.
+        """
+        source = text_type(source)
+        lines = source.splitlines()
+        if self.keep_trailing_newline and source:
+            for newline in ('\r\n', '\r', '\n'):
+                if source.endswith(newline):
+                    lines.append('')
+                    break
+        source = '\n'.join(lines)
+        pos = 0
+        lineno = 1
+        stack = ['root']
+        if state is not None and state != 'root':
+            assert state in ('variable', 'block'), 'invalid state'
+            stack.append(state + '_begin')
+        else:
+            state = 'root'
+        statetokens = self.rules[stack[-1]]
+        source_length = len(source)
+
+        balancing_stack = []
+
+        while 1:
+            # tokenizer loop
+            for regex, tokens, new_state in statetokens:
+                m = regex.match(source, pos)
+                # if no match we try again with the next rule
+                if m is None:
+                    continue
+
+                # we only match blocks and variables if braces / parentheses
+                # are balanced. continue parsing with the lower rule which
+                # is the operator rule. do this only if the end tags look
+                # like operators
+                if balancing_stack and \
+                   tokens in ('variable_end', 'block_end',
+                              'linestatement_end'):
+                    continue
+
+                # tuples support more options
+                if isinstance(tokens, tuple):
+                    for idx, token in enumerate(tokens):
+                        # failure group
+                        if token.__class__ is Failure:
+                            raise token(lineno, filename)
+                        # bygroup is a bit more complex, in that case we
+                        # yield for the current token the first named
+                        # group that matched
+                        elif token == '#bygroup':
+                            for key, value in iteritems(m.groupdict()):
+                                if value is not None:
+                                    yield lineno, key, value
+                                    lineno += value.count('\n')
+                                    break
+                            else:
+                                raise RuntimeError('%r wanted to resolve '
+                                                   'the token dynamically'
+                                                   ' but no group matched'
+                                                   % regex)
+                        # normal group
+                        else:
+                            data = m.group(idx + 1)
+                            if data or token not in ignore_if_empty:
+                                yield lineno, token, data
+                            lineno += data.count('\n')
+
+                # strings as token just are yielded as it.
+                else:
+                    data = m.group()
+                    # update brace/parentheses balance
+                    if tokens == 'operator':
+                        if data == '{':
+                            balancing_stack.append('}')
+                        elif data == '(':
+                            balancing_stack.append(')')
+                        elif data == '[':
+                            balancing_stack.append(']')
+                        elif data in ('}', ')', ']'):
+                            if not balancing_stack:
+                                raise TemplateSyntaxError('unexpected \'%s\'' %
+                                                          data, lineno, name,
+                                                          filename)
+                            expected_op = balancing_stack.pop()
+                            if expected_op != data:
+                                raise TemplateSyntaxError('unexpected \'%s\', '
+                                                          'expected \'%s\'' %
+                                                          (data, expected_op),
+                                                          lineno, name,
+                                                          filename)
+                    # yield items
+                    if data or tokens not in ignore_if_empty:
+                        yield lineno, tokens, data
+                    lineno += data.count('\n')
+
+                # fetch new position into new variable so that we can check
+                # if there is a internal parsing error which would result
+                # in an infinite loop
+                pos2 = m.end()
+
+                # handle state changes
+                if new_state is not None:
+                    # remove the uppermost state
+                    if new_state == '#pop':
+                        stack.pop()
+                    # resolve the new state by group checking
+                    elif new_state == '#bygroup':
+                        for key, value in iteritems(m.groupdict()):
+                            if value is not None:
+                                stack.append(key)
+                                break
+                        else:
+                            raise RuntimeError('%r wanted to resolve the '
+                                               'new state dynamically but'
+                                               ' no group matched' %
+                                               regex)
+                    # direct state name given
+                    else:
+                        stack.append(new_state)
+                    statetokens = self.rules[stack[-1]]
+                # we are still at the same position and no stack change.
+                # this means a loop without break condition, avoid that and
+                # raise error
+                elif pos2 == pos:
+                    raise RuntimeError('%r yielded empty string without '
+                                       'stack change' % regex)
+                # publish new function and start again
+                pos = pos2
+                break
+            # if loop terminated without break we haven't found a single match
+            # either we are at the end of the file or we have a problem
+            else:
+                # end of text
+                if pos >= source_length:
+                    return
+                # something went wrong
+                raise TemplateSyntaxError('unexpected char %r at %d' %
+                                          (source[pos], pos), lineno,
+                                          name, filename)

python/jinja2/loaders.py

@@ -0,0 +1,481 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.loaders
+    ~~~~~~~~~~~~~~
+
+    Jinja loader classes.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import weakref
+from types import ModuleType
+from os import path
+from hashlib import sha1
+from jinja2.exceptions import TemplateNotFound
+from jinja2.utils import open_if_exists, internalcode
+from jinja2._compat import string_types, iteritems
+
+
+def split_template_path(template):
+    """Split a path into segments and perform a sanity check.  If it detects
+    '..' in the path it will raise a `TemplateNotFound` error.
+    """
+    pieces = []
+    for piece in template.split('/'):
+        if path.sep in piece \
+           or (path.altsep and path.altsep in piece) or \
+           piece == path.pardir:
+            raise TemplateNotFound(template)
+        elif piece and piece != '.':
+            pieces.append(piece)
+    return pieces
+
+
+class BaseLoader(object):
+    """Baseclass for all loaders.  Subclass this and override `get_source` to
+    implement a custom loading mechanism.  The environment provides a
+    `get_template` method that calls the loader's `load` method to get the
+    :class:`Template` object.
+
+    A very basic example for a loader that looks up templates on the file
+    system could look like this::
+
+        from jinja2 import BaseLoader, TemplateNotFound
+        from os.path import join, exists, getmtime
+
+        class MyLoader(BaseLoader):
+
+            def __init__(self, path):
+                self.path = path
+
+            def get_source(self, environment, template):
+                path = join(self.path, template)
+                if not exists(path):
+                    raise TemplateNotFound(template)
+                mtime = getmtime(path)
+                with file(path) as f:
+                    source = f.read().decode('utf-8')
+                return source, path, lambda: mtime == getmtime(path)
+    """
+
+    #: if set to `False` it indicates that the loader cannot provide access
+    #: to the source of templates.
+    #:
+    #: .. versionadded:: 2.4
+    has_source_access = True
+
+    def get_source(self, environment, template):
+        """Get the template source, filename and reload helper for a template.
+        It's passed the environment and template name and has to return a
+        tuple in the form ``(source, filename, uptodate)`` or raise a
+        `TemplateNotFound` error if it can't locate the template.
+
+        The source part of the returned tuple must be the source of the
+        template as unicode string or a ASCII bytestring.  The filename should
+        be the name of the file on the filesystem if it was loaded from there,
+        otherwise `None`.  The filename is used by python for the tracebacks
+        if no loader extension is used.
+
+        The last item in the tuple is the `uptodate` function.  If auto
+        reloading is enabled it's always called to check if the template
+        changed.  No arguments are passed so the function must store the
+        old state somewhere (for example in a closure).  If it returns `False`
+        the template will be reloaded.
+        """
+        if not self.has_source_access:
+            raise RuntimeError('%s cannot provide access to the source' %
+                               self.__class__.__name__)
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        """Iterates over all templates.  If the loader does not support that
+        it should raise a :exc:`TypeError` which is the default behavior.
+        """
+        raise TypeError('this loader cannot iterate over all templates')
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        """Loads a template.  This method looks up the template in the cache
+        or loads one by calling :meth:`get_source`.  Subclasses should not
+        override this method as loaders working on collections of other
+        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
+        will not call this method but `get_source` directly.
+        """
+        code = None
+        if globals is None:
+            globals = {}
+
+        # first we try to get the source for this template together
+        # with the filename and the uptodate function.
+        source, filename, uptodate = self.get_source(environment, name)
+
+        # try to load the code from the bytecode cache if there is a
+        # bytecode cache configured.
+        bcc = environment.bytecode_cache
+        if bcc is not None:
+            bucket = bcc.get_bucket(environment, name, filename, source)
+            code = bucket.code
+
+        # if we don't have code so far (not cached, no longer up to
+        # date) etc. we compile the template
+        if code is None:
+            code = environment.compile(source, name, filename)
+
+        # if the bytecode cache is available and the bucket doesn't
+        # have a code so far, we give the bucket the new code and put
+        # it back to the bytecode cache.
+        if bcc is not None and bucket.code is None:
+            bucket.code = code
+            bcc.set_bucket(bucket)
+
+        return environment.template_class.from_code(environment, code,
+                                                    globals, uptodate)
+
+
+class FileSystemLoader(BaseLoader):
+    """Loads templates from the file system.  This loader can find templates
+    in folders on the file system and is the preferred way to load them.
+
+    The loader takes the path to the templates as string, or if multiple
+    locations are wanted a list of them which is then looked up in the
+    given order::
+
+    >>> loader = FileSystemLoader('/path/to/templates')
+    >>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
+
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.
+
+    To follow symbolic links, set the *followlinks* parameter to ``True``::
+
+    >>> loader = FileSystemLoader('/path/to/templates', followlinks=True)
+
+    .. versionchanged:: 2.8+
+       The *followlinks* parameter was added.
+    """
+
+    def __init__(self, searchpath, encoding='utf-8', followlinks=False):
+        if isinstance(searchpath, string_types):
+            searchpath = [searchpath]
+        self.searchpath = list(searchpath)
+        self.encoding = encoding
+        self.followlinks = followlinks
+
+    def get_source(self, environment, template):
+        pieces = split_template_path(template)
+        for searchpath in self.searchpath:
+            filename = path.join(searchpath, *pieces)
+            f = open_if_exists(filename)
+            if f is None:
+                continue
+            try:
+                contents = f.read().decode(self.encoding)
+            finally:
+                f.close()
+
+            mtime = path.getmtime(filename)
+
+            def uptodate():
+                try:
+                    return path.getmtime(filename) == mtime
+                except OSError:
+                    return False
+            return contents, filename, uptodate
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        found = set()
+        for searchpath in self.searchpath:
+            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
+            for dirpath, dirnames, filenames in walk_dir:
+                for filename in filenames:
+                    template = os.path.join(dirpath, filename) \
+                        [len(searchpath):].strip(os.path.sep) \
+                                          .replace(os.path.sep, '/')
+                    if template[:2] == './':
+                        template = template[2:]
+                    if template not in found:
+                        found.add(template)
+        return sorted(found)
+
+
+class PackageLoader(BaseLoader):
+    """Load templates from python eggs or packages.  It is constructed with
+    the name of the python package and the path to the templates in that
+    package::
+
+        loader = PackageLoader('mypackage', 'views')
+
+    If the package path is not given, ``'templates'`` is assumed.
+
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.  Due to the nature
+    of eggs it's only possible to reload templates if the package was loaded
+    from the file system and not a zip file.
+    """
+
+    def __init__(self, package_name, package_path='templates',
+                 encoding='utf-8'):
+        from pkg_resources import DefaultProvider, ResourceManager, \
+                                  get_provider
+        provider = get_provider(package_name)
+        self.encoding = encoding
+        self.manager = ResourceManager()
+        self.filesystem_bound = isinstance(provider, DefaultProvider)
+        self.provider = provider
+        self.package_path = package_path
+
+    def get_source(self, environment, template):
+        pieces = split_template_path(template)
+        p = '/'.join((self.package_path,) + tuple(pieces))
+        if not self.provider.has_resource(p):
+            raise TemplateNotFound(template)
+
+        filename = uptodate = None
+        if self.filesystem_bound:
+            filename = self.provider.get_resource_filename(self.manager, p)
+            mtime = path.getmtime(filename)
+            def uptodate():
+                try:
+                    return path.getmtime(filename) == mtime
+                except OSError:
+                    return False
+
+        source = self.provider.get_resource_string(self.manager, p)
+        return source.decode(self.encoding), filename, uptodate
+
+    def list_templates(self):
+        path = self.package_path
+        if path[:2] == './':
+            path = path[2:]
+        elif path == '.':
+            path = ''
+        offset = len(path)
+        results = []
+        def _walk(path):
+            for filename in self.provider.resource_listdir(path):
+                fullname = path + '/' + filename
+                if self.provider.resource_isdir(fullname):
+                    _walk(fullname)
+                else:
+                    results.append(fullname[offset:].lstrip('/'))
+        _walk(path)
+        results.sort()
+        return results
+
+
+class DictLoader(BaseLoader):
+    """Loads a template from a python dict.  It's passed a dict of unicode
+    strings bound to template names.  This loader is useful for unittesting:
+
+    >>> loader = DictLoader({'index.html': 'source here'})
+
+    Because auto reloading is rarely useful this is disabled per default.
+    """
+
+    def __init__(self, mapping):
+        self.mapping = mapping
+
+    def get_source(self, environment, template):
+        if template in self.mapping:
+            source = self.mapping[template]
+            return source, None, lambda: source == self.mapping.get(template)
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        return sorted(self.mapping)
+
+
+class FunctionLoader(BaseLoader):
+    """A loader that is passed a function which does the loading.  The
+    function receives the name of the template and has to return either
+    an unicode string with the template source, a tuple in the form ``(source,
+    filename, uptodatefunc)`` or `None` if the template does not exist.
+
+    >>> def load_template(name):
+    ...     if name == 'index.html':
+    ...         return '...'
+    ...
+    >>> loader = FunctionLoader(load_template)
+
+    The `uptodatefunc` is a function that is called if autoreload is enabled
+    and has to return `True` if the template is still up to date.  For more
+    details have a look at :meth:`BaseLoader.get_source` which has the same
+    return value.
+    """
+
+    def __init__(self, load_func):
+        self.load_func = load_func
+
+    def get_source(self, environment, template):
+        rv = self.load_func(template)
+        if rv is None:
+            raise TemplateNotFound(template)
+        elif isinstance(rv, string_types):
+            return rv, None, None
+        return rv
+
+
+class PrefixLoader(BaseLoader):
+    """A loader that is passed a dict of loaders where each loader is bound
+    to a prefix.  The prefix is delimited from the template by a slash per
+    default, which can be changed by setting the `delimiter` argument to
+    something else::
+
+        loader = PrefixLoader({
+            'app1':     PackageLoader('mypackage.app1'),
+            'app2':     PackageLoader('mypackage.app2')
+        })
+
+    By loading ``'app1/index.html'`` the file from the app1 package is loaded,
+    by loading ``'app2/index.html'`` the file from the second.
+    """
+
+    def __init__(self, mapping, delimiter='/'):
+        self.mapping = mapping
+        self.delimiter = delimiter
+
+    def get_loader(self, template):
+        try:
+            prefix, name = template.split(self.delimiter, 1)
+            loader = self.mapping[prefix]
+        except (ValueError, KeyError):
+            raise TemplateNotFound(template)
+        return loader, name
+
+    def get_source(self, environment, template):
+        loader, name = self.get_loader(template)
+        try:
+            return loader.get_source(environment, name)
+        except TemplateNotFound:
+            # re-raise the exception with the correct filename here.
+            # (the one that includes the prefix)
+            raise TemplateNotFound(template)
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        loader, local_name = self.get_loader(name)
+        try:
+            return loader.load(environment, local_name, globals)
+        except TemplateNotFound:
+            # re-raise the exception with the correct filename here.
+            # (the one that includes the prefix)
+            raise TemplateNotFound(name)
+
+    def list_templates(self):
+        result = []
+        for prefix, loader in iteritems(self.mapping):
+            for template in loader.list_templates():
+                result.append(prefix + self.delimiter + template)
+        return result
+
+
+class ChoiceLoader(BaseLoader):
+    """This loader works like the `PrefixLoader` just that no prefix is
+    specified.  If a template could not be found by one loader the next one
+    is tried.
+
+    >>> loader = ChoiceLoader([
+    ...     FileSystemLoader('/path/to/user/templates'),
+    ...     FileSystemLoader('/path/to/system/templates')
+    ... ])
+
+    This is useful if you want to allow users to override builtin templates
+    from a different location.
+    """
+
+    def __init__(self, loaders):
+        self.loaders = loaders
+
+    def get_source(self, environment, template):
+        for loader in self.loaders:
+            try:
+                return loader.get_source(environment, template)
+            except TemplateNotFound:
+                pass
+        raise TemplateNotFound(template)
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        for loader in self.loaders:
+            try:
+                return loader.load(environment, name, globals)
+            except TemplateNotFound:
+                pass
+        raise TemplateNotFound(name)
+
+    def list_templates(self):
+        found = set()
+        for loader in self.loaders:
+            found.update(loader.list_templates())
+        return sorted(found)
+
+
+class _TemplateModule(ModuleType):
+    """Like a normal module but with support for weak references"""
+
+
+class ModuleLoader(BaseLoader):
+    """This loader loads templates from precompiled templates.
+
+    Example usage:
+
+    >>> loader = ChoiceLoader([
+    ...     ModuleLoader('/path/to/compiled/templates'),
+    ...     FileSystemLoader('/path/to/templates')
+    ... ])
+
+    Templates can be precompiled with :meth:`Environment.compile_templates`.
+    """
+
+    has_source_access = False
+
+    def __init__(self, path):
+        package_name = '_jinja2_module_templates_%x' % id(self)
+
+        # create a fake module that looks for the templates in the
+        # path given.
+        mod = _TemplateModule(package_name)
+        if isinstance(path, string_types):
+            path = [path]
+        else:
+            path = list(path)
+        mod.__path__ = path
+
+        sys.modules[package_name] = weakref.proxy(mod,
+            lambda x: sys.modules.pop(package_name, None))
+
+        # the only strong reference, the sys.modules entry is weak
+        # so that the garbage collector can remove it once the
+        # loader that created it goes out of business.
+        self.module = mod
+        self.package_name = package_name
+
+    @staticmethod
+    def get_template_key(name):
+        return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest()
+
+    @staticmethod
+    def get_module_filename(name):
+        return ModuleLoader.get_template_key(name) + '.py'
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        key = self.get_template_key(name)
+        module = '%s.%s' % (self.package_name, key)
+        mod = getattr(self.module, module, None)
+        if mod is None:
+            try:
+                mod = __import__(module, None, None, ['root'])
+            except ImportError:
+                raise TemplateNotFound(name)
+
+            # remove the entry from sys.modules, we only want the attribute
+            # on the module object we have stored on the loader.
+            sys.modules.pop(module, None)
+
+        return environment.template_class.from_module_dict(
+            environment, mod.__dict__, globals)

python/jinja2/meta.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.meta
+    ~~~~~~~~~~~
+
+    This module implements various functions that exposes information about
+    templates that might be interesting for various kinds of applications.
+
+    :copyright: (c) 2017 by the Jinja Team, see AUTHORS for more details.
+    :license: BSD, see LICENSE for more details.
+"""
+from jinja2 import nodes
+from jinja2.compiler import CodeGenerator
+from jinja2._compat import string_types, iteritems
+
+
+class TrackingCodeGenerator(CodeGenerator):
+    """We abuse the code generator for introspection."""
+
+    def __init__(self, environment):
+        CodeGenerator.__init__(self, environment, '<introspection>',
+                               '<introspection>')
+        self.undeclared_identifiers = set()
+
+    def write(self, x):
+        """Don't write."""
+
+    def enter_frame(self, frame):
+        """Remember all undeclared identifiers."""
+        CodeGenerator.enter_frame(self, frame)
+        for _, (action, param) in iteritems(frame.symbols.loads):
+            if action == 'resolve':
+                self.undeclared_identifiers.add(param)
+
+
+def find_undeclared_variables(ast):
+    """Returns a set of all variables in the AST that will be looked up from
+    the context at runtime.  Because at compile time it's not known which
+    variables will be used depending on the path the execution takes at
+    runtime, all variables are returned.
+
+    >>> from jinja2 import Environment, meta
+    >>> env = Environment()
+    >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
+    >>> meta.find_undeclared_variables(ast) == set(['bar'])
+    True
+
+    .. admonition:: Implementation
+
+       Internally the code generator is used for finding undeclared variables.
+       This is good to know because the code generator might raise a
+       :exc:`TemplateAssertionError` during compilation and as a matter of
+       fact this function can currently raise that exception as well.
+    """
+    codegen = TrackingCodeGenerator(ast.environment)
+    codegen.visit(ast)
+    return codegen.undeclared_identifiers
+
+
+def find_referenced_templates(ast):
+    """Finds all the referenced templates from the AST.  This will return an
+    iterator over all the hardcoded template extensions, inclusions and
+    imports.  If dynamic inheritance or inclusion is used, `None` will be
+    yielded.
+
+    >>> from jinja2 import Environment, meta
+    >>> env = Environment()
+    >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}')
+    >>> list(meta.find_referenced_templates(ast))
+    ['layout.html', None]
+
+    This function is useful for dependency tracking.  For example if you want
+    to rebuild parts of the website after a layout template has changed.
+    """
+    for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import,
+                              nodes.Include)):
+        if not isinstance(node.template, nodes.Const):
+            # a tuple with some non consts in there
+            if isinstance(node.template, (nodes.Tuple, nodes.List)):
+                for template_name in node.template.items:
+                    # something const, only yield the strings and ignore
+                    # non-string consts that really just make no sense
+                    if isinstance(template_name, nodes.Const):
+                        if isinstance(template_name.value, string_types):
+                            yield template_name.value
+                    # something dynamic in there
+                    else:
+                        yield None
+            # something dynamic we don't know about here
+            else:
+                yield None
+            continue
+        # constant is a basestring, direct template name
+        if isinstance(node.template.value, string_types):
+            yield node.template.value
+        # a tuple or list (latter *should* not happen) made of consts,
+        # yield the consts that are strings.  We could warn here for
+        # non string values
+        elif isinstance(node, nodes.Include) and \
+             isinstance(node.template.value, (tuple, list)):
+            for template_name in node.template.value:
+                if isinstance(template_name, string_types):
+                    yield template_name
+        # something else we don't care about, we could warn here
+        else:
+            yield None

python/jinja2/nativetypes.py

@@ -0,0 +1,220 @@
+import sys
+from ast import literal_eval
+from itertools import islice, chain
+from jinja2 import nodes
+from jinja2._compat import text_type
+from jinja2.compiler import CodeGenerator, has_safe_repr
+from jinja2.environment import Environment, Template
+from jinja2.utils import concat, escape
+
+
+def native_concat(nodes):
+    """Return a native Python type from the list of compiled nodes. If the
+    result is a single node, its value is returned. Otherwise, the nodes are
+    concatenated as strings. If the result can be parsed with
+    :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+    string is returned.
+    """
+    head = list(islice(nodes, 2))
+
+    if not head:
+        return None
+
+    if len(head) == 1:
+        out = head[0]
+    else:
+        out = u''.join([text_type(v) for v in chain(head, nodes)])
+
+    try:
+        return literal_eval(out)
+    except (ValueError, SyntaxError, MemoryError):
+        return out
+
+
+class NativeCodeGenerator(CodeGenerator):
+    """A code generator which avoids injecting ``to_string()`` calls around the
+    internal code Jinja uses to render templates.
+    """
+
+    def visit_Output(self, node, frame):
+        """Same as :meth:`CodeGenerator.visit_Output`, but do not call
+        ``to_string`` on output nodes in generated code.
+        """
+        if self.has_known_extends and frame.require_output_check:
+            return
+
+        finalize = self.environment.finalize
+        finalize_context = getattr(finalize, 'contextfunction', False)
+        finalize_eval = getattr(finalize, 'evalcontextfunction', False)
+        finalize_env = getattr(finalize, 'environmentfunction', False)
+
+        if finalize is not None:
+            if finalize_context or finalize_eval:
+                const_finalize = None
+            elif finalize_env:
+                def const_finalize(x):
+                    return finalize(self.environment, x)
+            else:
+                const_finalize = finalize
+        else:
+            def const_finalize(x):
+                return x
+
+        # If we are inside a frame that requires output checking, we do so.
+        outdent_later = False
+
+        if frame.require_output_check:
+            self.writeline('if parent_template is None:')
+            self.indent()
+            outdent_later = True
+
+        # Try to evaluate as many chunks as possible into a static string at
+        # compile time.
+        body = []
+
+        for child in node.nodes:
+            try:
+                if const_finalize is None:
+                    raise nodes.Impossible()
+
+                const = child.as_const(frame.eval_ctx)
+                if not has_safe_repr(const):
+                    raise nodes.Impossible()
+            except nodes.Impossible:
+                body.append(child)
+                continue
+
+            # the frame can't be volatile here, because otherwise the as_const
+            # function would raise an Impossible exception at that point
+            try:
+                if frame.eval_ctx.autoescape:
+                    if hasattr(const, '__html__'):
+                        const = const.__html__()
+                    else:
+                        const = escape(const)
+
+                const = const_finalize(const)
+            except Exception:
+                # if something goes wrong here we evaluate the node at runtime
+                # for easier debugging
+                body.append(child)
+                continue
+
+            if body and isinstance(body[-1], list):
+                body[-1].append(const)
+            else:
+                body.append([const])
+
+        # if we have less than 3 nodes or a buffer we yield or extend/append
+        if len(body) < 3 or frame.buffer is not None:
+            if frame.buffer is not None:
+                # for one item we append, for more we extend
+                if len(body) == 1:
+                    self.writeline('%s.append(' % frame.buffer)
+                else:
+                    self.writeline('%s.extend((' % frame.buffer)
+
+                self.indent()
+
+            for item in body:
+                if isinstance(item, list):
+                    val = repr(native_concat(item))
+
+                    if frame.buffer is None:
+                        self.writeline('yield ' + val)
+                    else:
+                        self.writeline(val + ',')
+                else:
+                    if frame.buffer is None:
+                        self.writeline('yield ', item)
+                    else:
+                        self.newline(item)
+
+                    close = 0
+
+                    if finalize is not None:
+                        self.write('environment.finalize(')
+
+                        if finalize_context:
+                            self.write('context, ')
+
+                        close += 1
+
+                    self.visit(item, frame)
+
+                    if close > 0:
+                        self.write(')' * close)
+
+                    if frame.buffer is not None:
+                        self.write(',')
+
+            if frame.buffer is not None:
+                # close the open parentheses
+                self.outdent()
+                self.writeline(len(body) == 1 and ')' or '))')
+
+        # otherwise we create a format string as this is faster in that case
+        else:
+            format = []
+            arguments = []
+
+            for item in body:
+                if isinstance(item, list):
+                    format.append(native_concat(item).replace('%', '%%'))
+                else:
+                    format.append('%s')
+                    arguments.append(item)
+
+            self.writeline('yield ')
+            self.write(repr(concat(format)) + ' % (')
+            self.indent()
+
+            for argument in arguments:
+                self.newline(argument)
+                close = 0
+
+                if finalize is not None:
+                    self.write('environment.finalize(')
+
+                    if finalize_context:
+                        self.write('context, ')
+                    elif finalize_eval:
+                        self.write('context.eval_ctx, ')
+                    elif finalize_env:
+                        self.write('environment, ')
+
+                    close += 1
+
+                self.visit(argument, frame)
+                self.write(')' * close + ', ')
+
+            self.outdent()
+            self.writeline(')')
+
+        if outdent_later:
+            self.outdent()
+
+
+class NativeTemplate(Template):
+    def render(self, *args, **kwargs):
+        """Render the template to produce a native Python type. If the result
+        is a single node, its value is returned. Otherwise, the nodes are
+        concatenated as strings. If the result can be parsed with
+        :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+        string is returned.
+        """
+        vars = dict(*args, **kwargs)
+
+        try:
+            return native_concat(self.root_render_func(self.new_context(vars)))
+        except Exception:
+            exc_info = sys.exc_info()
+
+        return self.environment.handle_exception(exc_info, True)
+
+
+class NativeEnvironment(Environment):
+    """An environment that renders templates to native Python types."""
+
+    code_generator_class = NativeCodeGenerator
+    template_class = NativeTemplate

python/jinja2/nodes.py

@@ -0,0 +1,999 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.nodes
+    ~~~~~~~~~~~~
+
+    This module implements additional nodes derived from the ast base node.
+
+    It also provides some node tree helper functions like `in_lineno` and
+    `get_nodes` used by the parser and translator in order to normalize
+    python and jinja nodes.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import types
+import operator
+
+from collections import deque
+from jinja2.utils import Markup
+from jinja2._compat import izip, with_metaclass, text_type, PY2
+
+
+#: the types we support for context functions
+_context_function_types = (types.FunctionType, types.MethodType)
+
+
+_binop_to_func = {
+    '*':        operator.mul,
+    '/':        operator.truediv,
+    '//':       operator.floordiv,
+    '**':       operator.pow,
+    '%':        operator.mod,
+    '+':        operator.add,
+    '-':        operator.sub
+}
+
+_uaop_to_func = {
+    'not':      operator.not_,
+    '+':        operator.pos,
+    '-':        operator.neg
+}
+
+_cmpop_to_func = {
+    'eq':       operator.eq,
+    'ne':       operator.ne,
+    'gt':       operator.gt,
+    'gteq':     operator.ge,
+    'lt':       operator.lt,
+    'lteq':     operator.le,
+    'in':       lambda a, b: a in b,
+    'notin':    lambda a, b: a not in b
+}
+
+
+class Impossible(Exception):
+    """Raised if the node could not perform a requested action."""
+
+
+class NodeType(type):
+    """A metaclass for nodes that handles the field and attribute
+    inheritance.  fields and attributes from the parent class are
+    automatically forwarded to the child."""
+
+    def __new__(cls, name, bases, d):
+        for attr in 'fields', 'attributes':
+            storage = []
+            storage.extend(getattr(bases[0], attr, ()))
+            storage.extend(d.get(attr, ()))
+            assert len(bases) == 1, 'multiple inheritance not allowed'
+            assert len(storage) == len(set(storage)), 'layout conflict'
+            d[attr] = tuple(storage)
+        d.setdefault('abstract', False)
+        return type.__new__(cls, name, bases, d)
+
+
+class EvalContext(object):
+    """Holds evaluation time information.  Custom attributes can be attached
+    to it in extensions.
+    """
+
+    def __init__(self, environment, template_name=None):
+        self.environment = environment
+        if callable(environment.autoescape):
+            self.autoescape = environment.autoescape(template_name)
+        else:
+            self.autoescape = environment.autoescape
+        self.volatile = False
+
+    def save(self):
+        return self.__dict__.copy()
+
+    def revert(self, old):
+        self.__dict__.clear()
+        self.__dict__.update(old)
+
+
+def get_eval_context(node, ctx):
+    if ctx is None:
+        if node.environment is None:
+            raise RuntimeError('if no eval context is passed, the '
+                               'node must have an attached '
+                               'environment.')
+        return EvalContext(node.environment)
+    return ctx
+
+
+class Node(with_metaclass(NodeType, object)):
+    """Baseclass for all Jinja2 nodes.  There are a number of nodes available
+    of different types.  There are four major types:
+
+    -   :class:`Stmt`: statements
+    -   :class:`Expr`: expressions
+    -   :class:`Helper`: helper nodes
+    -   :class:`Template`: the outermost wrapper node
+
+    All nodes have fields and attributes.  Fields may be other nodes, lists,
+    or arbitrary values.  Fields are passed to the constructor as regular
+    positional arguments, attributes as keyword arguments.  Each node has
+    two attributes: `lineno` (the line number of the node) and `environment`.
+    The `environment` attribute is set at the end of the parsing process for
+    all nodes automatically.
+    """
+    fields = ()
+    attributes = ('lineno', 'environment')
+    abstract = True
+
+    def __init__(self, *fields, **attributes):
+        if self.abstract:
+            raise TypeError('abstract nodes are not instanciable')
+        if fields:
+            if len(fields) != len(self.fields):
+                if not self.fields:
+                    raise TypeError('%r takes 0 arguments' %
+                                    self.__class__.__name__)
+                raise TypeError('%r takes 0 or %d argument%s' % (
+                    self.__class__.__name__,
+                    len(self.fields),
+                    len(self.fields) != 1 and 's' or ''
+                ))
+            for name, arg in izip(self.fields, fields):
+                setattr(self, name, arg)
+        for attr in self.attributes:
+            setattr(self, attr, attributes.pop(attr, None))
+        if attributes:
+            raise TypeError('unknown attribute %r' %
+                            next(iter(attributes)))
+
+    def iter_fields(self, exclude=None, only=None):
+        """This method iterates over all fields that are defined and yields
+        ``(key, value)`` tuples.  Per default all fields are returned, but
+        it's possible to limit that to some fields by providing the `only`
+        parameter or to exclude some using the `exclude` parameter.  Both
+        should be sets or tuples of field names.
+        """
+        for name in self.fields:
+            if (exclude is only is None) or \
+               (exclude is not None and name not in exclude) or \
+               (only is not None and name in only):
+                try:
+                    yield name, getattr(self, name)
+                except AttributeError:
+                    pass
+
+    def iter_child_nodes(self, exclude=None, only=None):
+        """Iterates over all direct child nodes of the node.  This iterates
+        over all fields and yields the values of they are nodes.  If the value
+        of a field is a list all the nodes in that list are returned.
+        """
+        for field, item in self.iter_fields(exclude, only):
+            if isinstance(item, list):
+                for n in item:
+                    if isinstance(n, Node):
+                        yield n
+            elif isinstance(item, Node):
+                yield item
+
+    def find(self, node_type):
+        """Find the first node of a given type.  If no such node exists the
+        return value is `None`.
+        """
+        for result in self.find_all(node_type):
+            return result
+
+    def find_all(self, node_type):
+        """Find all the nodes of a given type.  If the type is a tuple,
+        the check is performed for any of the tuple items.
+        """
+        for child in self.iter_child_nodes():
+            if isinstance(child, node_type):
+                yield child
+            for result in child.find_all(node_type):
+                yield result
+
+    def set_ctx(self, ctx):
+        """Reset the context of a node and all child nodes.  Per default the
+        parser will all generate nodes that have a 'load' context as it's the
+        most common one.  This method is used in the parser to set assignment
+        targets and other nodes to a store context.
+        """
+        todo = deque([self])
+        while todo:
+            node = todo.popleft()
+            if 'ctx' in node.fields:
+                node.ctx = ctx
+            todo.extend(node.iter_child_nodes())
+        return self
+
+    def set_lineno(self, lineno, override=False):
+        """Set the line numbers of the node and children."""
+        todo = deque([self])
+        while todo:
+            node = todo.popleft()
+            if 'lineno' in node.attributes:
+                if node.lineno is None or override:
+                    node.lineno = lineno
+            todo.extend(node.iter_child_nodes())
+        return self
+
+    def set_environment(self, environment):
+        """Set the environment for all nodes."""
+        todo = deque([self])
+        while todo:
+            node = todo.popleft()
+            node.environment = environment
+            todo.extend(node.iter_child_nodes())
+        return self
+
+    def __eq__(self, other):
+        return type(self) is type(other) and \
+               tuple(self.iter_fields()) == tuple(other.iter_fields())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    # Restore Python 2 hashing behavior on Python 3
+    __hash__ = object.__hash__
+
+    def __repr__(self):
+        return '%s(%s)' % (
+            self.__class__.__name__,
+            ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
+                      arg in self.fields)
+        )
+
+    def dump(self):
+        def _dump(node):
+            if not isinstance(node, Node):
+                buf.append(repr(node))
+                return
+
+            buf.append('nodes.%s(' % node.__class__.__name__)
+            if not node.fields:
+                buf.append(')')
+                return
+            for idx, field in enumerate(node.fields):
+                if idx:
+                    buf.append(', ')
+                value = getattr(node, field)
+                if isinstance(value, list):
+                    buf.append('[')
+                    for idx, item in enumerate(value):
+                        if idx:
+                            buf.append(', ')
+                        _dump(item)
+                    buf.append(']')
+                else:
+                    _dump(value)
+            buf.append(')')
+        buf = []
+        _dump(self)
+        return ''.join(buf)
+
+
+
+class Stmt(Node):
+    """Base node for all statements."""
+    abstract = True
+
+
+class Helper(Node):
+    """Nodes that exist in a specific context only."""
+    abstract = True
+
+
+class Template(Node):
+    """Node that represents a template.  This must be the outermost node that
+    is passed to the compiler.
+    """
+    fields = ('body',)
+
+
+class Output(Stmt):
+    """A node that holds multiple expressions which are then printed out.
+    This is used both for the `print` statement and the regular template data.
+    """
+    fields = ('nodes',)
+
+
+class Extends(Stmt):
+    """Represents an extends statement."""
+    fields = ('template',)
+
+
+class For(Stmt):
+    """The for loop.  `target` is the target for the iteration (usually a
+    :class:`Name` or :class:`Tuple`), `iter` the iterable.  `body` is a list
+    of nodes that are used as loop-body, and `else_` a list of nodes for the
+    `else` block.  If no else node exists it has to be an empty list.
+
+    For filtered nodes an expression can be stored as `test`, otherwise `None`.
+    """
+    fields = ('target', 'iter', 'body', 'else_', 'test', 'recursive')
+
+
+class If(Stmt):
+    """If `test` is true, `body` is rendered, else `else_`."""
+    fields = ('test', 'body', 'elif_', 'else_')
+
+
+class Macro(Stmt):
+    """A macro definition.  `name` is the name of the macro, `args` a list of
+    arguments and `defaults` a list of defaults if there are any.  `body` is
+    a list of nodes for the macro body.
+    """
+    fields = ('name', 'args', 'defaults', 'body')
+
+
+class CallBlock(Stmt):
+    """Like a macro without a name but a call instead.  `call` is called with
+    the unnamed macro as `caller` argument this node holds.
+    """
+    fields = ('call', 'args', 'defaults', 'body')
+
+
+class FilterBlock(Stmt):
+    """Node for filter sections."""
+    fields = ('body', 'filter')
+
+
+class With(Stmt):
+    """Specific node for with statements.  In older versions of Jinja the
+    with statement was implemented on the base of the `Scope` node instead.
+
+    .. versionadded:: 2.9.3
+    """
+    fields = ('targets', 'values', 'body')
+
+
+class Block(Stmt):
+    """A node that represents a block."""
+    fields = ('name', 'body', 'scoped')
+
+
+class Include(Stmt):
+    """A node that represents the include tag."""
+    fields = ('template', 'with_context', 'ignore_missing')
+
+
+class Import(Stmt):
+    """A node that represents the import tag."""
+    fields = ('template', 'target', 'with_context')
+
+
+class FromImport(Stmt):
+    """A node that represents the from import tag.  It's important to not
+    pass unsafe names to the name attribute.  The compiler translates the
+    attribute lookups directly into getattr calls and does *not* use the
+    subscript callback of the interface.  As exported variables may not
+    start with double underscores (which the parser asserts) this is not a
+    problem for regular Jinja code, but if this node is used in an extension
+    extra care must be taken.
+
+    The list of names may contain tuples if aliases are wanted.
+    """
+    fields = ('template', 'names', 'with_context')
+
+
+class ExprStmt(Stmt):
+    """A statement that evaluates an expression and discards the result."""
+    fields = ('node',)
+
+
+class Assign(Stmt):
+    """Assigns an expression to a target."""
+    fields = ('target', 'node')
+
+
+class AssignBlock(Stmt):
+    """Assigns a block to a target."""
+    fields = ('target', 'filter', 'body')
+
+
+class Expr(Node):
+    """Baseclass for all expressions."""
+    abstract = True
+
+    def as_const(self, eval_ctx=None):
+        """Return the value of the expression as constant or raise
+        :exc:`Impossible` if this was not possible.
+
+        An :class:`EvalContext` can be provided, if none is given
+        a default context is created which requires the nodes to have
+        an attached environment.
+
+        .. versionchanged:: 2.4
+           the `eval_ctx` parameter was added.
+        """
+        raise Impossible()
+
+    def can_assign(self):
+        """Check if it's possible to assign something to this node."""
+        return False
+
+
+class BinExpr(Expr):
+    """Baseclass for all binary expressions."""
+    fields = ('left', 'right')
+    operator = None
+    abstract = True
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        # intercepted operators cannot be folded at compile time
+        if self.environment.sandboxed and \
+           self.operator in self.environment.intercepted_binops:
+            raise Impossible()
+        f = _binop_to_func[self.operator]
+        try:
+            return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx))
+        except Exception:
+            raise Impossible()
+
+
+class UnaryExpr(Expr):
+    """Baseclass for all unary expressions."""
+    fields = ('node',)
+    operator = None
+    abstract = True
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        # intercepted operators cannot be folded at compile time
+        if self.environment.sandboxed and \
+           self.operator in self.environment.intercepted_unops:
+            raise Impossible()
+        f = _uaop_to_func[self.operator]
+        try:
+            return f(self.node.as_const(eval_ctx))
+        except Exception:
+            raise Impossible()
+
+
+class Name(Expr):
+    """Looks up a name or stores a value in a name.
+    The `ctx` of the node can be one of the following values:
+
+    -   `store`: store a value in the name
+    -   `load`: load that name
+    -   `param`: like `store` but if the name was defined as function parameter.
+    """
+    fields = ('name', 'ctx')
+
+    def can_assign(self):
+        return self.name not in ('true', 'false', 'none',
+                                 'True', 'False', 'None')
+
+
+class NSRef(Expr):
+    """Reference to a namespace value assignment"""
+    fields = ('name', 'attr')
+
+    def can_assign(self):
+        # We don't need any special checks here; NSRef assignments have a
+        # runtime check to ensure the target is a namespace object which will
+        # have been checked already as it is created using a normal assignment
+        # which goes through a `Name` node.
+        return True
+
+
+class Literal(Expr):
+    """Baseclass for literals."""
+    abstract = True
+
+
+class Const(Literal):
+    """All constant values.  The parser will return this node for simple
+    constants such as ``42`` or ``"foo"`` but it can be used to store more
+    complex values such as lists too.  Only constants with a safe
+    representation (objects where ``eval(repr(x)) == x`` is true).
+    """
+    fields = ('value',)
+
+    def as_const(self, eval_ctx=None):
+        rv = self.value
+        if PY2 and type(rv) is text_type and \
+           self.environment.policies['compiler.ascii_str']:
+            try:
+                rv = rv.encode('ascii')
+            except UnicodeError:
+                pass
+        return rv
+
+    @classmethod
+    def from_untrusted(cls, value, lineno=None, environment=None):
+        """Return a const object if the value is representable as
+        constant value in the generated code, otherwise it will raise
+        an `Impossible` exception.
+        """
+        from .compiler import has_safe_repr
+        if not has_safe_repr(value):
+            raise Impossible()
+        return cls(value, lineno=lineno, environment=environment)
+
+
+class TemplateData(Literal):
+    """A constant template string."""
+    fields = ('data',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        if eval_ctx.volatile:
+            raise Impossible()
+        if eval_ctx.autoescape:
+            return Markup(self.data)
+        return self.data
+
+
+class Tuple(Literal):
+    """For loop unpacking and some other things like multiple arguments
+    for subscripts.  Like for :class:`Name` `ctx` specifies if the tuple
+    is used for loading the names or storing.
+    """
+    fields = ('items', 'ctx')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return tuple(x.as_const(eval_ctx) for x in self.items)
+
+    def can_assign(self):
+        for item in self.items:
+            if not item.can_assign():
+                return False
+        return True
+
+
+class List(Literal):
+    """Any list literal such as ``[1, 2, 3]``"""
+    fields = ('items',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return [x.as_const(eval_ctx) for x in self.items]
+
+
+class Dict(Literal):
+    """Any dict literal such as ``{1: 2, 3: 4}``.  The items must be a list of
+    :class:`Pair` nodes.
+    """
+    fields = ('items',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return dict(x.as_const(eval_ctx) for x in self.items)
+
+
+class Pair(Helper):
+    """A key, value pair for dicts."""
+    fields = ('key', 'value')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx)
+
+
+class Keyword(Helper):
+    """A key, value pair for keyword arguments where key is a string."""
+    fields = ('key', 'value')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return self.key, self.value.as_const(eval_ctx)
+
+
+class CondExpr(Expr):
+    """A conditional expression (inline if expression).  (``{{
+    foo if bar else baz }}``)
+    """
+    fields = ('test', 'expr1', 'expr2')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        if self.test.as_const(eval_ctx):
+            return self.expr1.as_const(eval_ctx)
+
+        # if we evaluate to an undefined object, we better do that at runtime
+        if self.expr2 is None:
+            raise Impossible()
+
+        return self.expr2.as_const(eval_ctx)
+
+
+def args_as_const(node, eval_ctx):
+    args = [x.as_const(eval_ctx) for x in node.args]
+    kwargs = dict(x.as_const(eval_ctx) for x in node.kwargs)
+
+    if node.dyn_args is not None:
+        try:
+            args.extend(node.dyn_args.as_const(eval_ctx))
+        except Exception:
+            raise Impossible()
+
+    if node.dyn_kwargs is not None:
+        try:
+            kwargs.update(node.dyn_kwargs.as_const(eval_ctx))
+        except Exception:
+            raise Impossible()
+
+    return args, kwargs
+
+
+class Filter(Expr):
+    """This node applies a filter on an expression.  `name` is the name of
+    the filter, the rest of the fields are the same as for :class:`Call`.
+
+    If the `node` of a filter is `None` the contents of the last buffer are
+    filtered.  Buffers are created by macros and filter blocks.
+    """
+
+    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+
+        if eval_ctx.volatile or self.node is None:
+            raise Impossible()
+
+        # we have to be careful here because we call filter_ below.
+        # if this variable would be called filter, 2to3 would wrap the
+        # call in a list beause it is assuming we are talking about the
+        # builtin filter function here which no longer returns a list in
+        # python 3.  because of that, do not rename filter_ to filter!
+        filter_ = self.environment.filters.get(self.name)
+
+        if filter_ is None or getattr(filter_, 'contextfilter', False):
+            raise Impossible()
+
+        # We cannot constant handle async filters, so we need to make sure
+        # to not go down this path.
+        if (
+            eval_ctx.environment.is_async
+            and getattr(filter_, 'asyncfiltervariant', False)
+        ):
+            raise Impossible()
+
+        args, kwargs = args_as_const(self, eval_ctx)
+        args.insert(0, self.node.as_const(eval_ctx))
+
+        if getattr(filter_, 'evalcontextfilter', False):
+            args.insert(0, eval_ctx)
+        elif getattr(filter_, 'environmentfilter', False):
+            args.insert(0, self.environment)
+
+        try:
+            return filter_(*args, **kwargs)
+        except Exception:
+            raise Impossible()
+
+
+class Test(Expr):
+    """Applies a test on an expression.  `name` is the name of the test, the
+    rest of the fields are the same as for :class:`Call`.
+    """
+
+    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+    def as_const(self, eval_ctx=None):
+        test = self.environment.tests.get(self.name)
+
+        if test is None:
+            raise Impossible()
+
+        eval_ctx = get_eval_context(self, eval_ctx)
+        args, kwargs = args_as_const(self, eval_ctx)
+        args.insert(0, self.node.as_const(eval_ctx))
+
+        try:
+            return test(*args, **kwargs)
+        except Exception:
+            raise Impossible()
+
+
+class Call(Expr):
+    """Calls an expression.  `args` is a list of arguments, `kwargs` a list
+    of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args`
+    and `dyn_kwargs` has to be either `None` or a node that is used as
+    node for dynamic positional (``*args``) or keyword (``**kwargs``)
+    arguments.
+    """
+    fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+
+class Getitem(Expr):
+    """Get an attribute or item from an expression and prefer the item."""
+    fields = ('node', 'arg', 'ctx')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        if self.ctx != 'load':
+            raise Impossible()
+        try:
+            return self.environment.getitem(self.node.as_const(eval_ctx),
+                                            self.arg.as_const(eval_ctx))
+        except Exception:
+            raise Impossible()
+
+    def can_assign(self):
+        return False
+
+
+class Getattr(Expr):
+    """Get an attribute or item from an expression that is a ascii-only
+    bytestring and prefer the attribute.
+    """
+    fields = ('node', 'attr', 'ctx')
+
+    def as_const(self, eval_ctx=None):
+        if self.ctx != 'load':
+            raise Impossible()
+        try:
+            eval_ctx = get_eval_context(self, eval_ctx)
+            return self.environment.getattr(self.node.as_const(eval_ctx),
+                                            self.attr)
+        except Exception:
+            raise Impossible()
+
+    def can_assign(self):
+        return False
+
+
+class Slice(Expr):
+    """Represents a slice object.  This must only be used as argument for
+    :class:`Subscript`.
+    """
+    fields = ('start', 'stop', 'step')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        def const(obj):
+            if obj is None:
+                return None
+            return obj.as_const(eval_ctx)
+        return slice(const(self.start), const(self.stop), const(self.step))
+
+
+class Concat(Expr):
+    """Concatenates the list of expressions provided after converting them to
+    unicode.
+    """
+    fields = ('nodes',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return ''.join(text_type(x.as_const(eval_ctx)) for x in self.nodes)
+
+
+class Compare(Expr):
+    """Compares an expression with some other expressions.  `ops` must be a
+    list of :class:`Operand`\\s.
+    """
+    fields = ('expr', 'ops')
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        result = value = self.expr.as_const(eval_ctx)
+        try:
+            for op in self.ops:
+                new_value = op.expr.as_const(eval_ctx)
+                result = _cmpop_to_func[op.op](value, new_value)
+                value = new_value
+        except Exception:
+            raise Impossible()
+        return result
+
+
+class Operand(Helper):
+    """Holds an operator and an expression."""
+    fields = ('op', 'expr')
+
+if __debug__:
+    Operand.__doc__ += '\nThe following operators are available: ' + \
+        ', '.join(sorted('``%s``' % x for x in set(_binop_to_func) |
+                  set(_uaop_to_func) | set(_cmpop_to_func)))
+
+
+class Mul(BinExpr):
+    """Multiplies the left with the right node."""
+    operator = '*'
+
+
+class Div(BinExpr):
+    """Divides the left by the right node."""
+    operator = '/'
+
+
+class FloorDiv(BinExpr):
+    """Divides the left by the right node and truncates conver the
+    result into an integer by truncating.
+    """
+    operator = '//'
+
+
+class Add(BinExpr):
+    """Add the left to the right node."""
+    operator = '+'
+
+
+class Sub(BinExpr):
+    """Subtract the right from the left node."""
+    operator = '-'
+
+
+class Mod(BinExpr):
+    """Left modulo right."""
+    operator = '%'
+
+
+class Pow(BinExpr):
+    """Left to the power of right."""
+    operator = '**'
+
+
+class And(BinExpr):
+    """Short circuited AND."""
+    operator = 'and'
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx)
+
+
+class Or(BinExpr):
+    """Short circuited OR."""
+    operator = 'or'
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx)
+
+
+class Not(UnaryExpr):
+    """Negate the expression."""
+    operator = 'not'
+
+
+class Neg(UnaryExpr):
+    """Make the expression negative."""
+    operator = '-'
+
+
+class Pos(UnaryExpr):
+    """Make the expression positive (noop for most expressions)"""
+    operator = '+'
+
+
+# Helpers for extensions
+
+
+class EnvironmentAttribute(Expr):
+    """Loads an attribute from the environment object.  This is useful for
+    extensions that want to call a callback stored on the environment.
+    """
+    fields = ('name',)
+
+
+class ExtensionAttribute(Expr):
+    """Returns the attribute of an extension bound to the environment.
+    The identifier is the identifier of the :class:`Extension`.
+
+    This node is usually constructed by calling the
+    :meth:`~jinja2.ext.Extension.attr` method on an extension.
+    """
+    fields = ('identifier', 'name')
+
+
+class ImportedName(Expr):
+    """If created with an import name the import name is returned on node
+    access.  For example ``ImportedName('cgi.escape')`` returns the `escape`
+    function from the cgi module on evaluation.  Imports are optimized by the
+    compiler so there is no need to assign them to local variables.
+    """
+    fields = ('importname',)
+
+
+class InternalName(Expr):
+    """An internal name in the compiler.  You cannot create these nodes
+    yourself but the parser provides a
+    :meth:`~jinja2.parser.Parser.free_identifier` method that creates
+    a new identifier for you.  This identifier is not available from the
+    template and is not threated specially by the compiler.
+    """
+    fields = ('name',)
+
+    def __init__(self):
+        raise TypeError('Can\'t create internal names.  Use the '
+                        '`free_identifier` method on a parser.')
+
+
+class MarkSafe(Expr):
+    """Mark the wrapped expression as safe (wrap it as `Markup`)."""
+    fields = ('expr',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        return Markup(self.expr.as_const(eval_ctx))
+
+
+class MarkSafeIfAutoescape(Expr):
+    """Mark the wrapped expression as safe (wrap it as `Markup`) but
+    only if autoescaping is active.
+
+    .. versionadded:: 2.5
+    """
+    fields = ('expr',)
+
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
+        if eval_ctx.volatile:
+            raise Impossible()
+        expr = self.expr.as_const(eval_ctx)
+        if eval_ctx.autoescape:
+            return Markup(expr)
+        return expr
+
+
+class ContextReference(Expr):
+    """Returns the current template context.  It can be used like a
+    :class:`Name` node, with a ``'load'`` ctx and will return the
+    current :class:`~jinja2.runtime.Context` object.
+
+    Here an example that assigns the current template name to a
+    variable named `foo`::
+
+        Assign(Name('foo', ctx='store'),
+               Getattr(ContextReference(), 'name'))
+    """
+
+
+class Continue(Stmt):
+    """Continue a loop."""
+
+
+class Break(Stmt):
+    """Break a loop."""
+
+
+class Scope(Stmt):
+    """An artificial scope."""
+    fields = ('body',)
+
+
+class OverlayScope(Stmt):
+    """An overlay scope for extensions.  This is a largely unoptimized scope
+    that however can be used to introduce completely arbitrary variables into
+    a sub scope from a dictionary or dictionary like object.  The `context`
+    field has to evaluate to a dictionary object.
+
+    Example usage::
+
+        OverlayScope(context=self.call_method('get_context'),
+                     body=[...])
+
+    .. versionadded:: 2.10
+    """
+    fields = ('context', 'body')
+
+
+class EvalContextModifier(Stmt):
+    """Modifies the eval context.  For each option that should be modified,
+    a :class:`Keyword` has to be added to the :attr:`options` list.
+
+    Example to change the `autoescape` setting::
+
+        EvalContextModifier(options=[Keyword('autoescape', Const(True))])
+    """
+    fields = ('options',)
+
+
+class ScopedEvalContextModifier(EvalContextModifier):
+    """Modifies the eval context and reverts it later.  Works exactly like
+    :class:`EvalContextModifier` but will only modify the
+    :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`.
+    """
+    fields = ('body',)
+
+
+# make sure nobody creates custom nodes
+def _failing_new(*args, **kwargs):
+    raise TypeError('can\'t create custom node types')
+NodeType.__new__ = staticmethod(_failing_new); del _failing_new

python/jinja2/optimizer.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.optimizer
+    ~~~~~~~~~~~~~~~~
+
+    The jinja optimizer is currently trying to constant fold a few expressions
+    and modify the AST in place so that it should be easier to evaluate it.
+
+    Because the AST does not contain all the scoping information and the
+    compiler has to find that out, we cannot do all the optimizations we
+    want.  For example loop unrolling doesn't work because unrolled loops would
+    have a different scoping.
+
+    The solution would be a second syntax tree that has the scoping rules stored.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+from jinja2 import nodes
+from jinja2.visitor import NodeTransformer
+
+
+def optimize(node, environment):
+    """The context hint can be used to perform an static optimization
+    based on the context given."""
+    optimizer = Optimizer(environment)
+    return optimizer.visit(node)
+
+
+class Optimizer(NodeTransformer):
+
+    def __init__(self, environment):
+        self.environment = environment
+
+    def fold(self, node, eval_ctx=None):
+        """Do constant folding."""
+        node = self.generic_visit(node)
+        try:
+            return nodes.Const.from_untrusted(node.as_const(eval_ctx),
+                                              lineno=node.lineno,
+                                              environment=self.environment)
+        except nodes.Impossible:
+            return node
+
+    visit_Add = visit_Sub = visit_Mul = visit_Div = visit_FloorDiv = \
+    visit_Pow = visit_Mod = visit_And = visit_Or = visit_Pos = visit_Neg = \
+    visit_Not = visit_Compare = visit_Getitem = visit_Getattr = visit_Call = \
+    visit_Filter = visit_Test = visit_CondExpr = fold
+    del fold

python/jinja2/parser.py

@@ -0,0 +1,903 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.parser
+    ~~~~~~~~~~~~~
+
+    Implements the template parser.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+from jinja2 import nodes
+from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
+from jinja2.lexer import describe_token, describe_token_expr
+from jinja2._compat import imap
+
+
+_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
+                                 'macro', 'include', 'from', 'import',
+                                 'set', 'with', 'autoescape'])
+_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
+
+_math_nodes = {
+    'add': nodes.Add,
+    'sub': nodes.Sub,
+    'mul': nodes.Mul,
+    'div': nodes.Div,
+    'floordiv': nodes.FloorDiv,
+    'mod': nodes.Mod,
+}
+
+
+class Parser(object):
+    """This is the central parsing class Jinja2 uses.  It's passed to
+    extensions and can be used to parse expressions or statements.
+    """
+
+    def __init__(self, environment, source, name=None, filename=None,
+                 state=None):
+        self.environment = environment
+        self.stream = environment._tokenize(source, name, filename, state)
+        self.name = name
+        self.filename = filename
+        self.closed = False
+        self.extensions = {}
+        for extension in environment.iter_extensions():
+            for tag in extension.tags:
+                self.extensions[tag] = extension.parse
+        self._last_identifier = 0
+        self._tag_stack = []
+        self._end_token_stack = []
+
+    def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
+        """Convenience method that raises `exc` with the message, passed
+        line number or last line number as well as the current name and
+        filename.
+        """
+        if lineno is None:
+            lineno = self.stream.current.lineno
+        raise exc(msg, lineno, self.name, self.filename)
+
+    def _fail_ut_eof(self, name, end_token_stack, lineno):
+        expected = []
+        for exprs in end_token_stack:
+            expected.extend(imap(describe_token_expr, exprs))
+        if end_token_stack:
+            currently_looking = ' or '.join(
+                "'%s'" % describe_token_expr(expr)
+                for expr in end_token_stack[-1])
+        else:
+            currently_looking = None
+
+        if name is None:
+            message = ['Unexpected end of template.']
+        else:
+            message = ['Encountered unknown tag \'%s\'.' % name]
+
+        if currently_looking:
+            if name is not None and name in expected:
+                message.append('You probably made a nesting mistake. Jinja '
+                               'is expecting this tag, but currently looking '
+                               'for %s.' % currently_looking)
+            else:
+                message.append('Jinja was looking for the following tags: '
+                               '%s.' % currently_looking)
+
+        if self._tag_stack:
+            message.append('The innermost block that needs to be '
+                           'closed is \'%s\'.' % self._tag_stack[-1])
+
+        self.fail(' '.join(message), lineno)
+
+    def fail_unknown_tag(self, name, lineno=None):
+        """Called if the parser encounters an unknown tag.  Tries to fail
+        with a human readable error message that could help to identify
+        the problem.
+        """
+        return self._fail_ut_eof(name, self._end_token_stack, lineno)
+
+    def fail_eof(self, end_tokens=None, lineno=None):
+        """Like fail_unknown_tag but for end of template situations."""
+        stack = list(self._end_token_stack)
+        if end_tokens is not None:
+            stack.append(end_tokens)
+        return self._fail_ut_eof(None, stack, lineno)
+
+    def is_tuple_end(self, extra_end_rules=None):
+        """Are we at the end of a tuple?"""
+        if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
+            return True
+        elif extra_end_rules is not None:
+            return self.stream.current.test_any(extra_end_rules)
+        return False
+
+    def free_identifier(self, lineno=None):
+        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
+        self._last_identifier += 1
+        rv = object.__new__(nodes.InternalName)
+        nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
+        return rv
+
+    def parse_statement(self):
+        """Parse a single statement."""
+        token = self.stream.current
+        if token.type != 'name':
+            self.fail('tag name expected', token.lineno)
+        self._tag_stack.append(token.value)
+        pop_tag = True
+        try:
+            if token.value in _statement_keywords:
+                return getattr(self, 'parse_' + self.stream.current.value)()
+            if token.value == 'call':
+                return self.parse_call_block()
+            if token.value == 'filter':
+                return self.parse_filter_block()
+            ext = self.extensions.get(token.value)
+            if ext is not None:
+                return ext(self)
+
+            # did not work out, remove the token we pushed by accident
+            # from the stack so that the unknown tag fail function can
+            # produce a proper error message.
+            self._tag_stack.pop()
+            pop_tag = False
+            self.fail_unknown_tag(token.value, token.lineno)
+        finally:
+            if pop_tag:
+                self._tag_stack.pop()
+
+    def parse_statements(self, end_tokens, drop_needle=False):
+        """Parse multiple statements into a list until one of the end tokens
+        is reached.  This is used to parse the body of statements as it also
+        parses template data if appropriate.  The parser checks first if the
+        current token is a colon and skips it if there is one.  Then it checks
+        for the block end and parses until if one of the `end_tokens` is
+        reached.  Per default the active token in the stream at the end of
+        the call is the matched end token.  If this is not wanted `drop_needle`
+        can be set to `True` and the end token is removed.
+        """
+        # the first token may be a colon for python compatibility
+        self.stream.skip_if('colon')
+
+        # in the future it would be possible to add whole code sections
+        # by adding some sort of end of statement token and parsing those here.
+        self.stream.expect('block_end')
+        result = self.subparse(end_tokens)
+
+        # we reached the end of the template too early, the subparser
+        # does not check for this, so we do that now
+        if self.stream.current.type == 'eof':
+            self.fail_eof(end_tokens)
+
+        if drop_needle:
+            next(self.stream)
+        return result
+
+    def parse_set(self):
+        """Parse an assign statement."""
+        lineno = next(self.stream).lineno
+        target = self.parse_assign_target(with_namespace=True)
+        if self.stream.skip_if('assign'):
+            expr = self.parse_tuple()
+            return nodes.Assign(target, expr, lineno=lineno)
+        filter_node = self.parse_filter(None)
+        body = self.parse_statements(('name:endset',),
+                                     drop_needle=True)
+        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
+
+    def parse_for(self):
+        """Parse a for loop."""
+        lineno = self.stream.expect('name:for').lineno
+        target = self.parse_assign_target(extra_end_rules=('name:in',))
+        self.stream.expect('name:in')
+        iter = self.parse_tuple(with_condexpr=False,
+                                extra_end_rules=('name:recursive',))
+        test = None
+        if self.stream.skip_if('name:if'):
+            test = self.parse_expression()
+        recursive = self.stream.skip_if('name:recursive')
+        body = self.parse_statements(('name:endfor', 'name:else'))
+        if next(self.stream).value == 'endfor':
+            else_ = []
+        else:
+            else_ = self.parse_statements(('name:endfor',), drop_needle=True)
+        return nodes.For(target, iter, body, else_, test,
+                         recursive, lineno=lineno)
+
+    def parse_if(self):
+        """Parse an if construct."""
+        node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
+        while 1:
+            node.test = self.parse_tuple(with_condexpr=False)
+            node.body = self.parse_statements(('name:elif', 'name:else',
+                                               'name:endif'))
+            node.elif_ = []
+            node.else_ = []
+            token = next(self.stream)
+            if token.test('name:elif'):
+                node = nodes.If(lineno=self.stream.current.lineno)
+                result.elif_.append(node)
+                continue
+            elif token.test('name:else'):
+                result.else_ = self.parse_statements(('name:endif',),
+                                                     drop_needle=True)
+            break
+        return result
+
+    def parse_with(self):
+        node = nodes.With(lineno=next(self.stream).lineno)
+        targets = []
+        values = []
+        while self.stream.current.type != 'block_end':
+            lineno = self.stream.current.lineno
+            if targets:
+                self.stream.expect('comma')
+            target = self.parse_assign_target()
+            target.set_ctx('param')
+            targets.append(target)
+            self.stream.expect('assign')
+            values.append(self.parse_expression())
+        node.targets = targets
+        node.values = values
+        node.body = self.parse_statements(('name:endwith',),
+                                          drop_needle=True)
+        return node
+
+    def parse_autoescape(self):
+        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
+        node.options = [
+            nodes.Keyword('autoescape', self.parse_expression())
+        ]
+        node.body = self.parse_statements(('name:endautoescape',),
+                                            drop_needle=True)
+        return nodes.Scope([node])
+
+    def parse_block(self):
+        node = nodes.Block(lineno=next(self.stream).lineno)
+        node.name = self.stream.expect('name').value
+        node.scoped = self.stream.skip_if('name:scoped')
+
+        # common problem people encounter when switching from django
+        # to jinja.  we do not support hyphens in block names, so let's
+        # raise a nicer error message in that case.
+        if self.stream.current.type == 'sub':
+            self.fail('Block names in Jinja have to be valid Python '
+                      'identifiers and may not contain hyphens, use an '
+                      'underscore instead.')
+
+        node.body = self.parse_statements(('name:endblock',), drop_needle=True)
+        self.stream.skip_if('name:' + node.name)
+        return node
+
+    def parse_extends(self):
+        node = nodes.Extends(lineno=next(self.stream).lineno)
+        node.template = self.parse_expression()
+        return node
+
+    def parse_import_context(self, node, default):
+        if self.stream.current.test_any('name:with', 'name:without') and \
+           self.stream.look().test('name:context'):
+            node.with_context = next(self.stream).value == 'with'
+            self.stream.skip()
+        else:
+            node.with_context = default
+        return node
+
+    def parse_include(self):
+        node = nodes.Include(lineno=next(self.stream).lineno)
+        node.template = self.parse_expression()
+        if self.stream.current.test('name:ignore') and \
+           self.stream.look().test('name:missing'):
+            node.ignore_missing = True
+            self.stream.skip(2)
+        else:
+            node.ignore_missing = False
+        return self.parse_import_context(node, True)
+
+    def parse_import(self):
+        node = nodes.Import(lineno=next(self.stream).lineno)
+        node.template = self.parse_expression()
+        self.stream.expect('name:as')
+        node.target = self.parse_assign_target(name_only=True).name
+        return self.parse_import_context(node, False)
+
+    def parse_from(self):
+        node = nodes.FromImport(lineno=next(self.stream).lineno)
+        node.template = self.parse_expression()
+        self.stream.expect('name:import')
+        node.names = []
+
+        def parse_context():
+            if self.stream.current.value in ('with', 'without') and \
+               self.stream.look().test('name:context'):
+                node.with_context = next(self.stream).value == 'with'
+                self.stream.skip()
+                return True
+            return False
+
+        while 1:
+            if node.names:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'name':
+                if parse_context():
+                    break
+                target = self.parse_assign_target(name_only=True)
+                if target.name.startswith('_'):
+                    self.fail('names starting with an underline can not '
+                              'be imported', target.lineno,
+                              exc=TemplateAssertionError)
+                if self.stream.skip_if('name:as'):
+                    alias = self.parse_assign_target(name_only=True)
+                    node.names.append((target.name, alias.name))
+                else:
+                    node.names.append(target.name)
+                if parse_context() or self.stream.current.type != 'comma':
+                    break
+            else:
+                self.stream.expect('name')
+        if not hasattr(node, 'with_context'):
+            node.with_context = False
+        return node
+
+    def parse_signature(self, node):
+        node.args = args = []
+        node.defaults = defaults = []
+        self.stream.expect('lparen')
+        while self.stream.current.type != 'rparen':
+            if args:
+                self.stream.expect('comma')
+            arg = self.parse_assign_target(name_only=True)
+            arg.set_ctx('param')
+            if self.stream.skip_if('assign'):
+                defaults.append(self.parse_expression())
+            elif defaults:
+                self.fail('non-default argument follows default argument')
+            args.append(arg)
+        self.stream.expect('rparen')
+
+    def parse_call_block(self):
+        node = nodes.CallBlock(lineno=next(self.stream).lineno)
+        if self.stream.current.type == 'lparen':
+            self.parse_signature(node)
+        else:
+            node.args = []
+            node.defaults = []
+
+        node.call = self.parse_expression()
+        if not isinstance(node.call, nodes.Call):
+            self.fail('expected call', node.lineno)
+        node.body = self.parse_statements(('name:endcall',), drop_needle=True)
+        return node
+
+    def parse_filter_block(self):
+        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
+        node.filter = self.parse_filter(None, start_inline=True)
+        node.body = self.parse_statements(('name:endfilter',),
+                                          drop_needle=True)
+        return node
+
+    def parse_macro(self):
+        node = nodes.Macro(lineno=next(self.stream).lineno)
+        node.name = self.parse_assign_target(name_only=True).name
+        self.parse_signature(node)
+        node.body = self.parse_statements(('name:endmacro',),
+                                          drop_needle=True)
+        return node
+
+    def parse_print(self):
+        node = nodes.Output(lineno=next(self.stream).lineno)
+        node.nodes = []
+        while self.stream.current.type != 'block_end':
+            if node.nodes:
+                self.stream.expect('comma')
+            node.nodes.append(self.parse_expression())
+        return node
+
+    def parse_assign_target(self, with_tuple=True, name_only=False,
+                            extra_end_rules=None, with_namespace=False):
+        """Parse an assignment target.  As Jinja2 allows assignments to
+        tuples, this function can parse all allowed assignment targets.  Per
+        default assignments to tuples are parsed, that can be disable however
+        by setting `with_tuple` to `False`.  If only assignments to names are
+        wanted `name_only` can be set to `True`.  The `extra_end_rules`
+        parameter is forwarded to the tuple parsing function.  If
+        `with_namespace` is enabled, a namespace assignment may be parsed.
+        """
+        if with_namespace and self.stream.look().type == 'dot':
+            token = self.stream.expect('name')
+            next(self.stream)  # dot
+            attr = self.stream.expect('name')
+            target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
+        elif name_only:
+            token = self.stream.expect('name')
+            target = nodes.Name(token.value, 'store', lineno=token.lineno)
+        else:
+            if with_tuple:
+                target = self.parse_tuple(simplified=True,
+                                          extra_end_rules=extra_end_rules)
+            else:
+                target = self.parse_primary()
+            target.set_ctx('store')
+        if not target.can_assign():
+            self.fail('can\'t assign to %r' % target.__class__.
+                      __name__.lower(), target.lineno)
+        return target
+
+    def parse_expression(self, with_condexpr=True):
+        """Parse an expression.  Per default all expressions are parsed, if
+        the optional `with_condexpr` parameter is set to `False` conditional
+        expressions are not parsed.
+        """
+        if with_condexpr:
+            return self.parse_condexpr()
+        return self.parse_or()
+
+    def parse_condexpr(self):
+        lineno = self.stream.current.lineno
+        expr1 = self.parse_or()
+        while self.stream.skip_if('name:if'):
+            expr2 = self.parse_or()
+            if self.stream.skip_if('name:else'):
+                expr3 = self.parse_condexpr()
+            else:
+                expr3 = None
+            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return expr1
+
+    def parse_or(self):
+        lineno = self.stream.current.lineno
+        left = self.parse_and()
+        while self.stream.skip_if('name:or'):
+            right = self.parse_and()
+            left = nodes.Or(left, right, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return left
+
+    def parse_and(self):
+        lineno = self.stream.current.lineno
+        left = self.parse_not()
+        while self.stream.skip_if('name:and'):
+            right = self.parse_not()
+            left = nodes.And(left, right, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return left
+
+    def parse_not(self):
+        if self.stream.current.test('name:not'):
+            lineno = next(self.stream).lineno
+            return nodes.Not(self.parse_not(), lineno=lineno)
+        return self.parse_compare()
+
+    def parse_compare(self):
+        lineno = self.stream.current.lineno
+        expr = self.parse_math1()
+        ops = []
+        while 1:
+            token_type = self.stream.current.type
+            if token_type in _compare_operators:
+                next(self.stream)
+                ops.append(nodes.Operand(token_type, self.parse_math1()))
+            elif self.stream.skip_if('name:in'):
+                ops.append(nodes.Operand('in', self.parse_math1()))
+            elif (self.stream.current.test('name:not') and
+                  self.stream.look().test('name:in')):
+                self.stream.skip(2)
+                ops.append(nodes.Operand('notin', self.parse_math1()))
+            else:
+                break
+            lineno = self.stream.current.lineno
+        if not ops:
+            return expr
+        return nodes.Compare(expr, ops, lineno=lineno)
+
+    def parse_math1(self):
+        lineno = self.stream.current.lineno
+        left = self.parse_concat()
+        while self.stream.current.type in ('add', 'sub'):
+            cls = _math_nodes[self.stream.current.type]
+            next(self.stream)
+            right = self.parse_concat()
+            left = cls(left, right, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return left
+
+    def parse_concat(self):
+        lineno = self.stream.current.lineno
+        args = [self.parse_math2()]
+        while self.stream.current.type == 'tilde':
+            next(self.stream)
+            args.append(self.parse_math2())
+        if len(args) == 1:
+            return args[0]
+        return nodes.Concat(args, lineno=lineno)
+
+    def parse_math2(self):
+        lineno = self.stream.current.lineno
+        left = self.parse_pow()
+        while self.stream.current.type in ('mul', 'div', 'floordiv', 'mod'):
+            cls = _math_nodes[self.stream.current.type]
+            next(self.stream)
+            right = self.parse_pow()
+            left = cls(left, right, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return left
+
+    def parse_pow(self):
+        lineno = self.stream.current.lineno
+        left = self.parse_unary()
+        while self.stream.current.type == 'pow':
+            next(self.stream)
+            right = self.parse_unary()
+            left = nodes.Pow(left, right, lineno=lineno)
+            lineno = self.stream.current.lineno
+        return left
+
+    def parse_unary(self, with_filter=True):
+        token_type = self.stream.current.type
+        lineno = self.stream.current.lineno
+        if token_type == 'sub':
+            next(self.stream)
+            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
+        elif token_type == 'add':
+            next(self.stream)
+            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
+        else:
+            node = self.parse_primary()
+        node = self.parse_postfix(node)
+        if with_filter:
+            node = self.parse_filter_expr(node)
+        return node
+
+    def parse_primary(self):
+        token = self.stream.current
+        if token.type == 'name':
+            if token.value in ('true', 'false', 'True', 'False'):
+                node = nodes.Const(token.value in ('true', 'True'),
+                                   lineno=token.lineno)
+            elif token.value in ('none', 'None'):
+                node = nodes.Const(None, lineno=token.lineno)
+            else:
+                node = nodes.Name(token.value, 'load', lineno=token.lineno)
+            next(self.stream)
+        elif token.type == 'string':
+            next(self.stream)
+            buf = [token.value]
+            lineno = token.lineno
+            while self.stream.current.type == 'string':
+                buf.append(self.stream.current.value)
+                next(self.stream)
+            node = nodes.Const(''.join(buf), lineno=lineno)
+        elif token.type in ('integer', 'float'):
+            next(self.stream)
+            node = nodes.Const(token.value, lineno=token.lineno)
+        elif token.type == 'lparen':
+            next(self.stream)
+            node = self.parse_tuple(explicit_parentheses=True)
+            self.stream.expect('rparen')
+        elif token.type == 'lbracket':
+            node = self.parse_list()
+        elif token.type == 'lbrace':
+            node = self.parse_dict()
+        else:
+            self.fail("unexpected '%s'" % describe_token(token), token.lineno)
+        return node
+
+    def parse_tuple(self, simplified=False, with_condexpr=True,
+                    extra_end_rules=None, explicit_parentheses=False):
+        """Works like `parse_expression` but if multiple expressions are
+        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
+        This method could also return a regular expression instead of a tuple
+        if no commas where found.
+
+        The default parsing mode is a full tuple.  If `simplified` is `True`
+        only names and literals are parsed.  The `no_condexpr` parameter is
+        forwarded to :meth:`parse_expression`.
+
+        Because tuples do not require delimiters and may end in a bogus comma
+        an extra hint is needed that marks the end of a tuple.  For example
+        for loops support tuples between `for` and `in`.  In that case the
+        `extra_end_rules` is set to ``['name:in']``.
+
+        `explicit_parentheses` is true if the parsing was triggered by an
+        expression in parentheses.  This is used to figure out if an empty
+        tuple is a valid expression or not.
+        """
+        lineno = self.stream.current.lineno
+        if simplified:
+            parse = self.parse_primary
+        elif with_condexpr:
+            parse = self.parse_expression
+        else:
+            parse = lambda: self.parse_expression(with_condexpr=False)
+        args = []
+        is_tuple = False
+        while 1:
+            if args:
+                self.stream.expect('comma')
+            if self.is_tuple_end(extra_end_rules):
+                break
+            args.append(parse())
+            if self.stream.current.type == 'comma':
+                is_tuple = True
+            else:
+                break
+            lineno = self.stream.current.lineno
+
+        if not is_tuple:
+            if args:
+                return args[0]
+
+            # if we don't have explicit parentheses, an empty tuple is
+            # not a valid expression.  This would mean nothing (literally
+            # nothing) in the spot of an expression would be an empty
+            # tuple.
+            if not explicit_parentheses:
+                self.fail('Expected an expression, got \'%s\'' %
+                          describe_token(self.stream.current))
+
+        return nodes.Tuple(args, 'load', lineno=lineno)
+
+    def parse_list(self):
+        token = self.stream.expect('lbracket')
+        items = []
+        while self.stream.current.type != 'rbracket':
+            if items:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'rbracket':
+                break
+            items.append(self.parse_expression())
+        self.stream.expect('rbracket')
+        return nodes.List(items, lineno=token.lineno)
+
+    def parse_dict(self):
+        token = self.stream.expect('lbrace')
+        items = []
+        while self.stream.current.type != 'rbrace':
+            if items:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'rbrace':
+                break
+            key = self.parse_expression()
+            self.stream.expect('colon')
+            value = self.parse_expression()
+            items.append(nodes.Pair(key, value, lineno=key.lineno))
+        self.stream.expect('rbrace')
+        return nodes.Dict(items, lineno=token.lineno)
+
+    def parse_postfix(self, node):
+        while 1:
+            token_type = self.stream.current.type
+            if token_type == 'dot' or token_type == 'lbracket':
+                node = self.parse_subscript(node)
+            # calls are valid both after postfix expressions (getattr
+            # and getitem) as well as filters and tests
+            elif token_type == 'lparen':
+                node = self.parse_call(node)
+            else:
+                break
+        return node
+
+    def parse_filter_expr(self, node):
+        while 1:
+            token_type = self.stream.current.type
+            if token_type == 'pipe':
+                node = self.parse_filter(node)
+            elif token_type == 'name' and self.stream.current.value == 'is':
+                node = self.parse_test(node)
+            # calls are valid both after postfix expressions (getattr
+            # and getitem) as well as filters and tests
+            elif token_type == 'lparen':
+                node = self.parse_call(node)
+            else:
+                break
+        return node
+
+    def parse_subscript(self, node):
+        token = next(self.stream)
+        if token.type == 'dot':
+            attr_token = self.stream.current
+            next(self.stream)
+            if attr_token.type == 'name':
+                return nodes.Getattr(node, attr_token.value, 'load',
+                                     lineno=token.lineno)
+            elif attr_token.type != 'integer':
+                self.fail('expected name or number', attr_token.lineno)
+            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
+            return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
+        if token.type == 'lbracket':
+            args = []
+            while self.stream.current.type != 'rbracket':
+                if args:
+                    self.stream.expect('comma')
+                args.append(self.parse_subscribed())
+            self.stream.expect('rbracket')
+            if len(args) == 1:
+                arg = args[0]
+            else:
+                arg = nodes.Tuple(args, 'load', lineno=token.lineno)
+            return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
+        self.fail('expected subscript expression', self.lineno)
+
+    def parse_subscribed(self):
+        lineno = self.stream.current.lineno
+
+        if self.stream.current.type == 'colon':
+            next(self.stream)
+            args = [None]
+        else:
+            node = self.parse_expression()
+            if self.stream.current.type != 'colon':
+                return node
+            next(self.stream)
+            args = [node]
+
+        if self.stream.current.type == 'colon':
+            args.append(None)
+        elif self.stream.current.type not in ('rbracket', 'comma'):
+            args.append(self.parse_expression())
+        else:
+            args.append(None)
+
+        if self.stream.current.type == 'colon':
+            next(self.stream)
+            if self.stream.current.type not in ('rbracket', 'comma'):
+                args.append(self.parse_expression())
+            else:
+                args.append(None)
+        else:
+            args.append(None)
+
+        return nodes.Slice(lineno=lineno, *args)
+
+    def parse_call(self, node):
+        token = self.stream.expect('lparen')
+        args = []
+        kwargs = []
+        dyn_args = dyn_kwargs = None
+        require_comma = False
+
+        def ensure(expr):
+            if not expr:
+                self.fail('invalid syntax for function call expression',
+                          token.lineno)
+
+        while self.stream.current.type != 'rparen':
+            if require_comma:
+                self.stream.expect('comma')
+                # support for trailing comma
+                if self.stream.current.type == 'rparen':
+                    break
+            if self.stream.current.type == 'mul':
+                ensure(dyn_args is None and dyn_kwargs is None)
+                next(self.stream)
+                dyn_args = self.parse_expression()
+            elif self.stream.current.type == 'pow':
+                ensure(dyn_kwargs is None)
+                next(self.stream)
+                dyn_kwargs = self.parse_expression()
+            else:
+                ensure(dyn_args is None and dyn_kwargs is None)
+                if self.stream.current.type == 'name' and \
+                   self.stream.look().type == 'assign':
+                    key = self.stream.current.value
+                    self.stream.skip(2)
+                    value = self.parse_expression()
+                    kwargs.append(nodes.Keyword(key, value,
+                                                lineno=value.lineno))
+                else:
+                    ensure(not kwargs)
+                    args.append(self.parse_expression())
+
+            require_comma = True
+        self.stream.expect('rparen')
+
+        if node is None:
+            return args, kwargs, dyn_args, dyn_kwargs
+        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
+                          lineno=token.lineno)
+
+    def parse_filter(self, node, start_inline=False):
+        while self.stream.current.type == 'pipe' or start_inline:
+            if not start_inline:
+                next(self.stream)
+            token = self.stream.expect('name')
+            name = token.value
+            while self.stream.current.type == 'dot':
+                next(self.stream)
+                name += '.' + self.stream.expect('name').value
+            if self.stream.current.type == 'lparen':
+                args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
+            else:
+                args = []
+                kwargs = []
+                dyn_args = dyn_kwargs = None
+            node = nodes.Filter(node, name, args, kwargs, dyn_args,
+                                dyn_kwargs, lineno=token.lineno)
+            start_inline = False
+        return node
+
+    def parse_test(self, node):
+        token = next(self.stream)
+        if self.stream.current.test('name:not'):
+            next(self.stream)
+            negated = True
+        else:
+            negated = False
+        name = self.stream.expect('name').value
+        while self.stream.current.type == 'dot':
+            next(self.stream)
+            name += '.' + self.stream.expect('name').value
+        dyn_args = dyn_kwargs = None
+        kwargs = []
+        if self.stream.current.type == 'lparen':
+            args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
+        elif (self.stream.current.type in ('name', 'string', 'integer',
+                                           'float', 'lparen', 'lbracket',
+                                           'lbrace') and not
+              self.stream.current.test_any('name:else', 'name:or',
+                                           'name:and')):
+            if self.stream.current.test('name:is'):
+                self.fail('You cannot chain multiple tests with is')
+            args = [self.parse_primary()]
+        else:
+            args = []
+        node = nodes.Test(node, name, args, kwargs, dyn_args,
+                          dyn_kwargs, lineno=token.lineno)
+        if negated:
+            node = nodes.Not(node, lineno=token.lineno)
+        return node
+
+    def subparse(self, end_tokens=None):
+        body = []
+        data_buffer = []
+        add_data = data_buffer.append
+
+        if end_tokens is not None:
+            self._end_token_stack.append(end_tokens)
+
+        def flush_data():
+            if data_buffer:
+                lineno = data_buffer[0].lineno
+                body.append(nodes.Output(data_buffer[:], lineno=lineno))
+                del data_buffer[:]
+
+        try:
+            while self.stream:
+                token = self.stream.current
+                if token.type == 'data':
+                    if token.value:
+                        add_data(nodes.TemplateData(token.value,
+                                                    lineno=token.lineno))
+                    next(self.stream)
+                elif token.type == 'variable_begin':
+                    next(self.stream)
+                    add_data(self.parse_tuple(with_condexpr=True))
+                    self.stream.expect('variable_end')
+                elif token.type == 'block_begin':
+                    flush_data()
+                    next(self.stream)
+                    if end_tokens is not None and \
+                       self.stream.current.test_any(*end_tokens):
+                        return body
+                    rv = self.parse_statement()
+                    if isinstance(rv, list):
+                        body.extend(rv)
+                    else:
+                        body.append(rv)
+                    self.stream.expect('block_end')
+                else:
+                    raise AssertionError('internal parsing error')
+
+            flush_data()
+        finally:
+            if end_tokens is not None:
+                self._end_token_stack.pop()
+
+        return body
+
+    def parse(self):
+        """Parse the whole template into a `Template` node."""
+        result = nodes.Template(self.subparse(), lineno=1)
+        result.set_environment(self.environment)
+        return result

python/jinja2/runtime.py

@@ -0,0 +1,813 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.runtime
+    ~~~~~~~~~~~~~~
+
+    Runtime helpers.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+import sys
+
+from itertools import chain
+from types import MethodType
+
+from jinja2.nodes import EvalContext, _context_function_types
+from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \
+     internalcode, object_type_repr, evalcontextfunction, Namespace
+from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \
+     TemplateNotFound
+from jinja2._compat import imap, text_type, iteritems, \
+     implements_iterator, implements_to_string, string_types, PY2, \
+     with_metaclass
+
+
+# these variables are exported to the template runtime
+__all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup',
+           'TemplateRuntimeError', 'missing', 'concat', 'escape',
+           'markup_join', 'unicode_join', 'to_string', 'identity',
+           'TemplateNotFound', 'Namespace']
+
+#: the name of the function that is used to convert something into
+#: a string.  We can just use the text type here.
+to_string = text_type
+
+#: the identity function.  Useful for certain things in the environment
+identity = lambda x: x
+
+_first_iteration = object()
+_last_iteration = object()
+
+
+def markup_join(seq):
+    """Concatenation that escapes if necessary and converts to unicode."""
+    buf = []
+    iterator = imap(soft_unicode, seq)
+    for arg in iterator:
+        buf.append(arg)
+        if hasattr(arg, '__html__'):
+            return Markup(u'').join(chain(buf, iterator))
+    return concat(buf)
+
+
+def unicode_join(seq):
+    """Simple args to unicode conversion and concatenation."""
+    return concat(imap(text_type, seq))
+
+
+def new_context(environment, template_name, blocks, vars=None,
+                shared=None, globals=None, locals=None):
+    """Internal helper to for context creation."""
+    if vars is None:
+        vars = {}
+    if shared:
+        parent = vars
+    else:
+        parent = dict(globals or (), **vars)
+    if locals:
+        # if the parent is shared a copy should be created because
+        # we don't want to modify the dict passed
+        if shared:
+            parent = dict(parent)
+        for key, value in iteritems(locals):
+            if value is not missing:
+                parent[key] = value
+    return environment.context_class(environment, parent, template_name,
+                                     blocks)
+
+
+class TemplateReference(object):
+    """The `self` in templates."""
+
+    def __init__(self, context):
+        self.__context = context
+
+    def __getitem__(self, name):
+        blocks = self.__context.blocks[name]
+        return BlockReference(name, self.__context, blocks, 0)
+
+    def __repr__(self):
+        return '<%s %r>' % (
+            self.__class__.__name__,
+            self.__context.name
+        )
+
+
+def _get_func(x):
+    return getattr(x, '__func__', x)
+
+
+class ContextMeta(type):
+
+    def __new__(cls, name, bases, d):
+        rv = type.__new__(cls, name, bases, d)
+        if bases == ():
+            return rv
+
+        resolve = _get_func(rv.resolve)
+        default_resolve = _get_func(Context.resolve)
+        resolve_or_missing = _get_func(rv.resolve_or_missing)
+        default_resolve_or_missing = _get_func(Context.resolve_or_missing)
+
+        # If we have a changed resolve but no changed default or missing
+        # resolve we invert the call logic.
+        if resolve is not default_resolve and \
+           resolve_or_missing is default_resolve_or_missing:
+            rv._legacy_resolve_mode = True
+        elif resolve is default_resolve and \
+             resolve_or_missing is default_resolve_or_missing:
+            rv._fast_resolve_mode = True
+
+        return rv
+
+
+def resolve_or_missing(context, key, missing=missing):
+    if key in context.vars:
+        return context.vars[key]
+    if key in context.parent:
+        return context.parent[key]
+    return missing
+
+
+class Context(with_metaclass(ContextMeta)):
+    """The template context holds the variables of a template.  It stores the
+    values passed to the template and also the names the template exports.
+    Creating instances is neither supported nor useful as it's created
+    automatically at various stages of the template evaluation and should not
+    be created by hand.
+
+    The context is immutable.  Modifications on :attr:`parent` **must not**
+    happen and modifications on :attr:`vars` are allowed from generated
+    template code only.  Template filters and global functions marked as
+    :func:`contextfunction`\\s get the active context passed as first argument
+    and are allowed to access the context read-only.
+
+    The template context supports read only dict operations (`get`,
+    `keys`, `values`, `items`, `iterkeys`, `itervalues`, `iteritems`,
+    `__getitem__`, `__contains__`).  Additionally there is a :meth:`resolve`
+    method that doesn't fail with a `KeyError` but returns an
+    :class:`Undefined` object for missing variables.
+    """
+    # XXX: we want to eventually make this be a deprecation warning and
+    # remove it.
+    _legacy_resolve_mode = False
+    _fast_resolve_mode = False
+
+    def __init__(self, environment, parent, name, blocks):
+        self.parent = parent
+        self.vars = {}
+        self.environment = environment
+        self.eval_ctx = EvalContext(self.environment, name)
+        self.exported_vars = set()
+        self.name = name
+
+        # create the initial mapping of blocks.  Whenever template inheritance
+        # takes place the runtime will update this mapping with the new blocks
+        # from the template.
+        self.blocks = dict((k, [v]) for k, v in iteritems(blocks))
+
+        # In case we detect the fast resolve mode we can set up an alias
+        # here that bypasses the legacy code logic.
+        if self._fast_resolve_mode:
+            self.resolve_or_missing = MethodType(resolve_or_missing, self)
+
+    def super(self, name, current):
+        """Render a parent block."""
+        try:
+            blocks = self.blocks[name]
+            index = blocks.index(current) + 1
+            blocks[index]
+        except LookupError:
+            return self.environment.undefined('there is no parent block '
+                                              'called %r.' % name,
+                                              name='super')
+        return BlockReference(name, self, blocks, index)
+
+    def get(self, key, default=None):
+        """Returns an item from the template context, if it doesn't exist
+        `default` is returned.
+        """
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def resolve(self, key):
+        """Looks up a variable like `__getitem__` or `get` but returns an
+        :class:`Undefined` object with the name of the name looked up.
+        """
+        if self._legacy_resolve_mode:
+            rv = resolve_or_missing(self, key)
+        else:
+            rv = self.resolve_or_missing(key)
+        if rv is missing:
+            return self.environment.undefined(name=key)
+        return rv
+
+    def resolve_or_missing(self, key):
+        """Resolves a variable like :meth:`resolve` but returns the
+        special `missing` value if it cannot be found.
+        """
+        if self._legacy_resolve_mode:
+            rv = self.resolve(key)
+            if isinstance(rv, Undefined):
+                rv = missing
+            return rv
+        return resolve_or_missing(self, key)
+
+    def get_exported(self):
+        """Get a new dict with the exported variables."""
+        return dict((k, self.vars[k]) for k in self.exported_vars)
+
+    def get_all(self):
+        """Return the complete context as dict including the exported
+        variables.  For optimizations reasons this might not return an
+        actual copy so be careful with using it.
+        """
+        if not self.vars:
+            return self.parent
+        if not self.parent:
+            return self.vars
+        return dict(self.parent, **self.vars)
+
+    @internalcode
+    def call(__self, __obj, *args, **kwargs):
+        """Call the callable with the arguments and keyword arguments
+        provided but inject the active context or environment as first
+        argument if the callable is a :func:`contextfunction` or
+        :func:`environmentfunction`.
+        """
+        if __debug__:
+            __traceback_hide__ = True  # noqa
+
+        # Allow callable classes to take a context
+        if hasattr(__obj, '__call__'):
+            fn = __obj.__call__
+            for fn_type in ('contextfunction',
+                            'evalcontextfunction',
+                            'environmentfunction'):
+                if hasattr(fn, fn_type):
+                    __obj = fn
+                    break
+
+        if isinstance(__obj, _context_function_types):
+            if getattr(__obj, 'contextfunction', 0):
+                args = (__self,) + args
+            elif getattr(__obj, 'evalcontextfunction', 0):
+                args = (__self.eval_ctx,) + args
+            elif getattr(__obj, 'environmentfunction', 0):
+                args = (__self.environment,) + args
+        try:
+            return __obj(*args, **kwargs)
+        except StopIteration:
+            return __self.environment.undefined('value was undefined because '
+                                                'a callable raised a '
+                                                'StopIteration exception')
+
+    def derived(self, locals=None):
+        """Internal helper function to create a derived context.  This is
+        used in situations where the system needs a new context in the same
+        template that is independent.
+        """
+        context = new_context(self.environment, self.name, {},
+                              self.get_all(), True, None, locals)
+        context.eval_ctx = self.eval_ctx
+        context.blocks.update((k, list(v)) for k, v in iteritems(self.blocks))
+        return context
+
+    def _all(meth):
+        proxy = lambda self: getattr(self.get_all(), meth)()
+        proxy.__doc__ = getattr(dict, meth).__doc__
+        proxy.__name__ = meth
+        return proxy
+
+    keys = _all('keys')
+    values = _all('values')
+    items = _all('items')
+
+    # not available on python 3
+    if PY2:
+        iterkeys = _all('iterkeys')
+        itervalues = _all('itervalues')
+        iteritems = _all('iteritems')
+    del _all
+
+    def __contains__(self, name):
+        return name in self.vars or name in self.parent
+
+    def __getitem__(self, key):
+        """Lookup a variable or raise `KeyError` if the variable is
+        undefined.
+        """
+        item = self.resolve_or_missing(key)
+        if item is missing:
+            raise KeyError(key)
+        return item
+
+    def __repr__(self):
+        return '<%s %s of %r>' % (
+            self.__class__.__name__,
+            repr(self.get_all()),
+            self.name
+        )
+
+
+# register the context as mapping if possible
+try:
+    from collections import Mapping
+    Mapping.register(Context)
+except ImportError:
+    pass
+
+
+class BlockReference(object):
+    """One block on a template reference."""
+
+    def __init__(self, name, context, stack, depth):
+        self.name = name
+        self._context = context
+        self._stack = stack
+        self._depth = depth
+
+    @property
+    def super(self):
+        """Super the block."""
+        if self._depth + 1 >= len(self._stack):
+            return self._context.environment. \
+                undefined('there is no parent block called %r.' %
+                          self.name, name='super')
+        return BlockReference(self.name, self._context, self._stack,
+                              self._depth + 1)
+
+    @internalcode
+    def __call__(self):
+        rv = concat(self._stack[self._depth](self._context))
+        if self._context.eval_ctx.autoescape:
+            rv = Markup(rv)
+        return rv
+
+
+class LoopContextBase(object):
+    """A loop context for dynamic iteration."""
+
+    _before = _first_iteration
+    _current = _first_iteration
+    _after = _last_iteration
+    _length = None
+
+    def __init__(self, undefined, recurse=None, depth0=0):
+        self._undefined = undefined
+        self._recurse = recurse
+        self.index0 = -1
+        self.depth0 = depth0
+        self._last_checked_value = missing
+
+    def cycle(self, *args):
+        """Cycles among the arguments with the current loop index."""
+        if not args:
+            raise TypeError('no items for cycling given')
+        return args[self.index0 % len(args)]
+
+    def changed(self, *value):
+        """Checks whether the value has changed since the last call."""
+        if self._last_checked_value != value:
+            self._last_checked_value = value
+            return True
+        return False
+
+    first = property(lambda x: x.index0 == 0)
+    last = property(lambda x: x._after is _last_iteration)
+    index = property(lambda x: x.index0 + 1)
+    revindex = property(lambda x: x.length - x.index0)
+    revindex0 = property(lambda x: x.length - x.index)
+    depth = property(lambda x: x.depth0 + 1)
+
+    @property
+    def previtem(self):
+        if self._before is _first_iteration:
+            return self._undefined('there is no previous item')
+        return self._before
+
+    @property
+    def nextitem(self):
+        if self._after is _last_iteration:
+            return self._undefined('there is no next item')
+        return self._after
+
+    def __len__(self):
+        return self.length
+
+    @internalcode
+    def loop(self, iterable):
+        if self._recurse is None:
+            raise TypeError('Tried to call non recursive loop.  Maybe you '
+                            "forgot the 'recursive' modifier.")
+        return self._recurse(iterable, self._recurse, self.depth0 + 1)
+
+    # a nifty trick to enhance the error message if someone tried to call
+    # the the loop without or with too many arguments.
+    __call__ = loop
+    del loop
+
+    def __repr__(self):
+        return '<%s %r/%r>' % (
+            self.__class__.__name__,
+            self.index,
+            self.length
+        )
+
+
+class LoopContext(LoopContextBase):
+
+    def __init__(self, iterable, undefined, recurse=None, depth0=0):
+        LoopContextBase.__init__(self, undefined, recurse, depth0)
+        self._iterator = iter(iterable)
+
+        # try to get the length of the iterable early.  This must be done
+        # here because there are some broken iterators around where there
+        # __len__ is the number of iterations left (i'm looking at your
+        # listreverseiterator!).
+        try:
+            self._length = len(iterable)
+        except (TypeError, AttributeError):
+            self._length = None
+        self._after = self._safe_next()
+
+    @property
+    def length(self):
+        if self._length is None:
+            # if was not possible to get the length of the iterator when
+            # the loop context was created (ie: iterating over a generator)
+            # we have to convert the iterable into a sequence and use the
+            # length of that + the number of iterations so far.
+            iterable = tuple(self._iterator)
+            self._iterator = iter(iterable)
+            iterations_done = self.index0 + 2
+            self._length = len(iterable) + iterations_done
+        return self._length
+
+    def __iter__(self):
+        return LoopContextIterator(self)
+
+    def _safe_next(self):
+        try:
+            return next(self._iterator)
+        except StopIteration:
+            return _last_iteration
+
+
+@implements_iterator
+class LoopContextIterator(object):
+    """The iterator for a loop context."""
+    __slots__ = ('context',)
+
+    def __init__(self, context):
+        self.context = context
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        ctx = self.context
+        ctx.index0 += 1
+        if ctx._after is _last_iteration:
+            raise StopIteration()
+        ctx._before = ctx._current
+        ctx._current = ctx._after
+        ctx._after = ctx._safe_next()
+        return ctx._current, ctx
+
+
+class Macro(object):
+    """Wraps a macro function."""
+
+    def __init__(self, environment, func, name, arguments,
+                 catch_kwargs, catch_varargs, caller,
+                 default_autoescape=None):
+        self._environment = environment
+        self._func = func
+        self._argument_count = len(arguments)
+        self.name = name
+        self.arguments = arguments
+        self.catch_kwargs = catch_kwargs
+        self.catch_varargs = catch_varargs
+        self.caller = caller
+        self.explicit_caller = 'caller' in arguments
+        if default_autoescape is None:
+            default_autoescape = environment.autoescape
+        self._default_autoescape = default_autoescape
+
+    @internalcode
+    @evalcontextfunction
+    def __call__(self, *args, **kwargs):
+        # This requires a bit of explanation,  In the past we used to
+        # decide largely based on compile-time information if a macro is
+        # safe or unsafe.  While there was a volatile mode it was largely
+        # unused for deciding on escaping.  This turns out to be
+        # problemtic for macros because if a macro is safe or not not so
+        # much depends on the escape mode when it was defined but when it
+        # was used.
+        #
+        # Because however we export macros from the module system and
+        # there are historic callers that do not pass an eval context (and
+        # will continue to not pass one), we need to perform an instance
+        # check here.
+        #
+        # This is considered safe because an eval context is not a valid
+        # argument to callables otherwise anwyays.  Worst case here is
+        # that if no eval context is passed we fall back to the compile
+        # time autoescape flag.
+        if args and isinstance(args[0], EvalContext):
+            autoescape = args[0].autoescape
+            args = args[1:]
+        else:
+            autoescape = self._default_autoescape
+
+        # try to consume the positional arguments
+        arguments = list(args[:self._argument_count])
+        off = len(arguments)
+
+        # For information why this is necessary refer to the handling
+        # of caller in the `macro_body` handler in the compiler.
+        found_caller = False
+
+        # if the number of arguments consumed is not the number of
+        # arguments expected we start filling in keyword arguments
+        # and defaults.
+        if off != self._argument_count:
+            for idx, name in enumerate(self.arguments[len(arguments):]):
+                try:
+                    value = kwargs.pop(name)
+                except KeyError:
+                    value = missing
+                if name == 'caller':
+                    found_caller = True
+                arguments.append(value)
+        else:
+            found_caller = self.explicit_caller
+
+        # it's important that the order of these arguments does not change
+        # if not also changed in the compiler's `function_scoping` method.
+        # the order is caller, keyword arguments, positional arguments!
+        if self.caller and not found_caller:
+            caller = kwargs.pop('caller', None)
+            if caller is None:
+                caller = self._environment.undefined('No caller defined',
+                                                     name='caller')
+            arguments.append(caller)
+
+        if self.catch_kwargs:
+            arguments.append(kwargs)
+        elif kwargs:
+            if 'caller' in kwargs:
+                raise TypeError('macro %r was invoked with two values for '
+                                'the special caller argument.  This is '
+                                'most likely a bug.' % self.name)
+            raise TypeError('macro %r takes no keyword argument %r' %
+                            (self.name, next(iter(kwargs))))
+        if self.catch_varargs:
+            arguments.append(args[self._argument_count:])
+        elif len(args) > self._argument_count:
+            raise TypeError('macro %r takes not more than %d argument(s)' %
+                            (self.name, len(self.arguments)))
+
+        return self._invoke(arguments, autoescape)
+
+    def _invoke(self, arguments, autoescape):
+        """This method is being swapped out by the async implementation."""
+        rv = self._func(*arguments)
+        if autoescape:
+            rv = Markup(rv)
+        return rv
+
+    def __repr__(self):
+        return '<%s %s>' % (
+            self.__class__.__name__,
+            self.name is None and 'anonymous' or repr(self.name)
+        )
+
+
+@implements_to_string
+class Undefined(object):
+    """The default undefined type.  This undefined type can be printed and
+    iterated over, but every other access will raise an :exc:`jinja2.exceptions.UndefinedError`:
+
+    >>> foo = Undefined(name='foo')
+    >>> str(foo)
+    ''
+    >>> not foo
+    True
+    >>> foo + 42
+    Traceback (most recent call last):
+      ...
+    jinja2.exceptions.UndefinedError: 'foo' is undefined
+    """
+    __slots__ = ('_undefined_hint', '_undefined_obj', '_undefined_name',
+                 '_undefined_exception')
+
+    def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError):
+        self._undefined_hint = hint
+        self._undefined_obj = obj
+        self._undefined_name = name
+        self._undefined_exception = exc
+
+    @internalcode
+    def _fail_with_undefined_error(self, *args, **kwargs):
+        """Regular callback function for undefined objects that raises an
+        `jinja2.exceptions.UndefinedError` on call.
+        """
+        if self._undefined_hint is None:
+            if self._undefined_obj is missing:
+                hint = '%r is undefined' % self._undefined_name
+            elif not isinstance(self._undefined_name, string_types):
+                hint = '%s has no element %r' % (
+                    object_type_repr(self._undefined_obj),
+                    self._undefined_name
+                )
+            else:
+                hint = '%r has no attribute %r' % (
+                    object_type_repr(self._undefined_obj),
+                    self._undefined_name
+                )
+        else:
+            hint = self._undefined_hint
+        raise self._undefined_exception(hint)
+
+    @internalcode
+    def __getattr__(self, name):
+        if name[:2] == '__':
+            raise AttributeError(name)
+        return self._fail_with_undefined_error()
+
+    __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \
+        __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \
+        __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \
+        __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \
+        __float__ = __complex__ = __pow__ = __rpow__ = __sub__ = \
+        __rsub__ = _fail_with_undefined_error
+
+    def __eq__(self, other):
+        return type(self) is type(other)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return id(type(self))
+
+    def __str__(self):
+        return u''
+
+    def __len__(self):
+        return 0
+
+    def __iter__(self):
+        if 0:
+            yield None
+
+    def __nonzero__(self):
+        return False
+    __bool__ = __nonzero__
+
+    def __repr__(self):
+        return 'Undefined'
+
+
+def make_logging_undefined(logger=None, base=None):
+    """Given a logger object this returns a new undefined class that will
+    log certain failures.  It will log iterations and printing.  If no
+    logger is given a default logger is created.
+
+    Example::
+
+        logger = logging.getLogger(__name__)
+        LoggingUndefined = make_logging_undefined(
+            logger=logger,
+            base=Undefined
+        )
+
+    .. versionadded:: 2.8
+
+    :param logger: the logger to use.  If not provided, a default logger
+                   is created.
+    :param base: the base class to add logging functionality to.  This
+                 defaults to :class:`Undefined`.
+    """
+    if logger is None:
+        import logging
+        logger = logging.getLogger(__name__)
+        logger.addHandler(logging.StreamHandler(sys.stderr))
+    if base is None:
+        base = Undefined
+
+    def _log_message(undef):
+        if undef._undefined_hint is None:
+            if undef._undefined_obj is missing:
+                hint = '%s is undefined' % undef._undefined_name
+            elif not isinstance(undef._undefined_name, string_types):
+                hint = '%s has no element %s' % (
+                    object_type_repr(undef._undefined_obj),
+                    undef._undefined_name)
+            else:
+                hint = '%s has no attribute %s' % (
+                    object_type_repr(undef._undefined_obj),
+                    undef._undefined_name)
+        else:
+            hint = undef._undefined_hint
+        logger.warning('Template variable warning: %s', hint)
+
+    class LoggingUndefined(base):
+
+        def _fail_with_undefined_error(self, *args, **kwargs):
+            try:
+                return base._fail_with_undefined_error(self, *args, **kwargs)
+            except self._undefined_exception as e:
+                logger.error('Template variable error: %s', str(e))
+                raise e
+
+        def __str__(self):
+            rv = base.__str__(self)
+            _log_message(self)
+            return rv
+
+        def __iter__(self):
+            rv = base.__iter__(self)
+            _log_message(self)
+            return rv
+
+        if PY2:
+            def __nonzero__(self):
+                rv = base.__nonzero__(self)
+                _log_message(self)
+                return rv
+
+            def __unicode__(self):
+                rv = base.__unicode__(self)
+                _log_message(self)
+                return rv
+        else:
+            def __bool__(self):
+                rv = base.__bool__(self)
+                _log_message(self)
+                return rv
+
+    return LoggingUndefined
+
+
+@implements_to_string
+class DebugUndefined(Undefined):
+    """An undefined that returns the debug info when printed.
+
+    >>> foo = DebugUndefined(name='foo')
+    >>> str(foo)
+    '{{ foo }}'
+    >>> not foo
+    True
+    >>> foo + 42
+    Traceback (most recent call last):
+      ...
+    jinja2.exceptions.UndefinedError: 'foo' is undefined
+    """
+    __slots__ = ()
+
+    def __str__(self):
+        if self._undefined_hint is None:
+            if self._undefined_obj is missing:
+                return u'{{ %s }}' % self._undefined_name
+            return '{{ no such element: %s[%r] }}' % (
+                object_type_repr(self._undefined_obj),
+                self._undefined_name
+            )
+        return u'{{ undefined value printed: %s }}' % self._undefined_hint
+
+
+@implements_to_string
+class StrictUndefined(Undefined):
+    """An undefined that barks on print and iteration as well as boolean
+    tests and all kinds of comparisons.  In other words: you can do nothing
+    with it except checking if it's defined using the `defined` test.
+
+    >>> foo = StrictUndefined(name='foo')
+    >>> str(foo)
+    Traceback (most recent call last):
+      ...
+    jinja2.exceptions.UndefinedError: 'foo' is undefined
+    >>> not foo
+    Traceback (most recent call last):
+      ...
+    jinja2.exceptions.UndefinedError: 'foo' is undefined
+    >>> foo + 42
+    Traceback (most recent call last):
+      ...
+    jinja2.exceptions.UndefinedError: 'foo' is undefined
+    """
+    __slots__ = ()
+    __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \
+        __ne__ = __bool__ = __hash__ = \
+        Undefined._fail_with_undefined_error
+
+
+# remove remaining slots attributes, after the metaclass did the magic they
+# are unneeded and irritating as they contain wrong data for the subclasses.
+del Undefined.__slots__, DebugUndefined.__slots__, StrictUndefined.__slots__

python/jinja2/sandbox.py

@@ -0,0 +1,486 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.sandbox
+    ~~~~~~~~~~~~~~
+
+    Adds a sandbox layer to Jinja as it was the default behavior in the old
+    Jinja 1 releases.  This sandbox is slightly different from Jinja 1 as the
+    default behavior is easier to use.
+
+    The behavior can be changed by subclassing the environment.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+import types
+import operator
+from collections import Mapping
+from jinja2.environment import Environment
+from jinja2.exceptions import SecurityError
+from jinja2._compat import string_types, PY2
+from jinja2.utils import Markup
+
+from markupsafe import EscapeFormatter
+from string import Formatter
+
+
+#: maximum number of items a range may produce
+MAX_RANGE = 100000
+
+#: attributes of function objects that are considered unsafe.
+if PY2:
+    UNSAFE_FUNCTION_ATTRIBUTES = set(['func_closure', 'func_code', 'func_dict',
+                                      'func_defaults', 'func_globals'])
+else:
+    # On versions > python 2 the special attributes on functions are gone,
+    # but they remain on methods and generators for whatever reason.
+    UNSAFE_FUNCTION_ATTRIBUTES = set()
+
+
+#: unsafe method attributes.  function attributes are unsafe for methods too
+UNSAFE_METHOD_ATTRIBUTES = set(['im_class', 'im_func', 'im_self'])
+
+#: unsafe generator attirbutes.
+UNSAFE_GENERATOR_ATTRIBUTES = set(['gi_frame', 'gi_code'])
+
+#: unsafe attributes on coroutines
+UNSAFE_COROUTINE_ATTRIBUTES = set(['cr_frame', 'cr_code'])
+
+#: unsafe attributes on async generators
+UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = set(['ag_code', 'ag_frame'])
+
+import warnings
+
+# make sure we don't warn in python 2.6 about stuff we don't care about
+warnings.filterwarnings('ignore', 'the sets module', DeprecationWarning,
+                        module='jinja2.sandbox')
+
+from collections import deque
+
+_mutable_set_types = (set,)
+_mutable_mapping_types = (dict,)
+_mutable_sequence_types = (list,)
+
+
+# on python 2.x we can register the user collection types
+try:
+    from UserDict import UserDict, DictMixin
+    from UserList import UserList
+    _mutable_mapping_types += (UserDict, DictMixin)
+    _mutable_set_types += (UserList,)
+except ImportError:
+    pass
+
+# if sets is still available, register the mutable set from there as well
+try:
+    from sets import Set
+    _mutable_set_types += (Set,)
+except ImportError:
+    pass
+
+#: register Python 2.6 abstract base classes
+from collections import MutableSet, MutableMapping, MutableSequence
+_mutable_set_types += (MutableSet,)
+_mutable_mapping_types += (MutableMapping,)
+_mutable_sequence_types += (MutableSequence,)
+
+
+_mutable_spec = (
+    (_mutable_set_types, frozenset([
+        'add', 'clear', 'difference_update', 'discard', 'pop', 'remove',
+        'symmetric_difference_update', 'update'
+    ])),
+    (_mutable_mapping_types, frozenset([
+        'clear', 'pop', 'popitem', 'setdefault', 'update'
+    ])),
+    (_mutable_sequence_types, frozenset([
+        'append', 'reverse', 'insert', 'sort', 'extend', 'remove'
+    ])),
+    (deque, frozenset([
+        'append', 'appendleft', 'clear', 'extend', 'extendleft', 'pop',
+        'popleft', 'remove', 'rotate'
+    ]))
+)
+
+
+class _MagicFormatMapping(Mapping):
+    """This class implements a dummy wrapper to fix a bug in the Python
+    standard library for string formatting.
+
+    See https://bugs.python.org/issue13598 for information about why
+    this is necessary.
+    """
+
+    def __init__(self, args, kwargs):
+        self._args = args
+        self._kwargs = kwargs
+        self._last_index = 0
+
+    def __getitem__(self, key):
+        if key == '':
+            idx = self._last_index
+            self._last_index += 1
+            try:
+                return self._args[idx]
+            except LookupError:
+                pass
+            key = str(idx)
+        return self._kwargs[key]
+
+    def __iter__(self):
+        return iter(self._kwargs)
+
+    def __len__(self):
+        return len(self._kwargs)
+
+
+def inspect_format_method(callable):
+    if not isinstance(callable, (types.MethodType,
+                                 types.BuiltinMethodType)) or \
+       callable.__name__ not in ('format', 'format_map'):
+        return None
+    obj = callable.__self__
+    if isinstance(obj, string_types):
+        return obj
+
+
+def safe_range(*args):
+    """A range that can't generate ranges with a length of more than
+    MAX_RANGE items.
+    """
+    rng = range(*args)
+    if len(rng) > MAX_RANGE:
+        raise OverflowError('range too big, maximum size for range is %d' %
+                            MAX_RANGE)
+    return rng
+
+
+def unsafe(f):
+    """Marks a function or method as unsafe.
+
+    ::
+
+        @unsafe
+        def delete(self):
+            pass
+    """
+    f.unsafe_callable = True
+    return f
+
+
+def is_internal_attribute(obj, attr):
+    """Test if the attribute given is an internal python attribute.  For
+    example this function returns `True` for the `func_code` attribute of
+    python objects.  This is useful if the environment method
+    :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden.
+
+    >>> from jinja2.sandbox import is_internal_attribute
+    >>> is_internal_attribute(str, "mro")
+    True
+    >>> is_internal_attribute(str, "upper")
+    False
+    """
+    if isinstance(obj, types.FunctionType):
+        if attr in UNSAFE_FUNCTION_ATTRIBUTES:
+            return True
+    elif isinstance(obj, types.MethodType):
+        if attr in UNSAFE_FUNCTION_ATTRIBUTES or \
+           attr in UNSAFE_METHOD_ATTRIBUTES:
+            return True
+    elif isinstance(obj, type):
+        if attr == 'mro':
+            return True
+    elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)):
+        return True
+    elif isinstance(obj, types.GeneratorType):
+        if attr in UNSAFE_GENERATOR_ATTRIBUTES:
+            return True
+    elif hasattr(types, 'CoroutineType') and isinstance(obj, types.CoroutineType):
+        if attr in UNSAFE_COROUTINE_ATTRIBUTES:
+            return True
+    elif hasattr(types, 'AsyncGeneratorType') and isinstance(obj, types.AsyncGeneratorType):
+        if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES:
+            return True
+    return attr.startswith('__')
+
+
+def modifies_known_mutable(obj, attr):
+    """This function checks if an attribute on a builtin mutable object
+    (list, dict, set or deque) would modify it if called.  It also supports
+    the "user"-versions of the objects (`sets.Set`, `UserDict.*` etc.) and
+    with Python 2.6 onwards the abstract base classes `MutableSet`,
+    `MutableMapping`, and `MutableSequence`.
+
+    >>> modifies_known_mutable({}, "clear")
+    True
+    >>> modifies_known_mutable({}, "keys")
+    False
+    >>> modifies_known_mutable([], "append")
+    True
+    >>> modifies_known_mutable([], "index")
+    False
+
+    If called with an unsupported object (such as unicode) `False` is
+    returned.
+
+    >>> modifies_known_mutable("foo", "upper")
+    False
+    """
+    for typespec, unsafe in _mutable_spec:
+        if isinstance(obj, typespec):
+            return attr in unsafe
+    return False
+
+
+class SandboxedEnvironment(Environment):
+    """The sandboxed environment.  It works like the regular environment but
+    tells the compiler to generate sandboxed code.  Additionally subclasses of
+    this environment may override the methods that tell the runtime what
+    attributes or functions are safe to access.
+
+    If the template tries to access insecure code a :exc:`SecurityError` is
+    raised.  However also other exceptions may occur during the rendering so
+    the caller has to ensure that all exceptions are caught.
+    """
+    sandboxed = True
+
+    #: default callback table for the binary operators.  A copy of this is
+    #: available on each instance of a sandboxed environment as
+    #: :attr:`binop_table`
+    default_binop_table = {
+        '+':        operator.add,
+        '-':        operator.sub,
+        '*':        operator.mul,
+        '/':        operator.truediv,
+        '//':       operator.floordiv,
+        '**':       operator.pow,
+        '%':        operator.mod
+    }
+
+    #: default callback table for the unary operators.  A copy of this is
+    #: available on each instance of a sandboxed environment as
+    #: :attr:`unop_table`
+    default_unop_table = {
+        '+':        operator.pos,
+        '-':        operator.neg
+    }
+
+    #: a set of binary operators that should be intercepted.  Each operator
+    #: that is added to this set (empty by default) is delegated to the
+    #: :meth:`call_binop` method that will perform the operator.  The default
+    #: operator callback is specified by :attr:`binop_table`.
+    #:
+    #: The following binary operators are interceptable:
+    #: ``//``, ``%``, ``+``, ``*``, ``-``, ``/``, and ``**``
+    #:
+    #: The default operation form the operator table corresponds to the
+    #: builtin function.  Intercepted calls are always slower than the native
+    #: operator call, so make sure only to intercept the ones you are
+    #: interested in.
+    #:
+    #: .. versionadded:: 2.6
+    intercepted_binops = frozenset()
+
+    #: a set of unary operators that should be intercepted.  Each operator
+    #: that is added to this set (empty by default) is delegated to the
+    #: :meth:`call_unop` method that will perform the operator.  The default
+    #: operator callback is specified by :attr:`unop_table`.
+    #:
+    #: The following unary operators are interceptable: ``+``, ``-``
+    #:
+    #: The default operation form the operator table corresponds to the
+    #: builtin function.  Intercepted calls are always slower than the native
+    #: operator call, so make sure only to intercept the ones you are
+    #: interested in.
+    #:
+    #: .. versionadded:: 2.6
+    intercepted_unops = frozenset()
+
+    def intercept_unop(self, operator):
+        """Called during template compilation with the name of a unary
+        operator to check if it should be intercepted at runtime.  If this
+        method returns `True`, :meth:`call_unop` is excuted for this unary
+        operator.  The default implementation of :meth:`call_unop` will use
+        the :attr:`unop_table` dictionary to perform the operator with the
+        same logic as the builtin one.
+
+        The following unary operators are interceptable: ``+`` and ``-``
+
+        Intercepted calls are always slower than the native operator call,
+        so make sure only to intercept the ones you are interested in.
+
+        .. versionadded:: 2.6
+        """
+        return False
+
+
+    def __init__(self, *args, **kwargs):
+        Environment.__init__(self, *args, **kwargs)
+        self.globals['range'] = safe_range
+        self.binop_table = self.default_binop_table.copy()
+        self.unop_table = self.default_unop_table.copy()
+
+    def is_safe_attribute(self, obj, attr, value):
+        """The sandboxed environment will call this method to check if the
+        attribute of an object is safe to access.  Per default all attributes
+        starting with an underscore are considered private as well as the
+        special attributes of internal python objects as returned by the
+        :func:`is_internal_attribute` function.
+        """
+        return not (attr.startswith('_') or is_internal_attribute(obj, attr))
+
+    def is_safe_callable(self, obj):
+        """Check if an object is safely callable.  Per default a function is
+        considered safe unless the `unsafe_callable` attribute exists and is
+        True.  Override this method to alter the behavior, but this won't
+        affect the `unsafe` decorator from this module.
+        """
+        return not (getattr(obj, 'unsafe_callable', False) or
+                    getattr(obj, 'alters_data', False))
+
+    def call_binop(self, context, operator, left, right):
+        """For intercepted binary operator calls (:meth:`intercepted_binops`)
+        this function is executed instead of the builtin operator.  This can
+        be used to fine tune the behavior of certain operators.
+
+        .. versionadded:: 2.6
+        """
+        return self.binop_table[operator](left, right)
+
+    def call_unop(self, context, operator, arg):
+        """For intercepted unary operator calls (:meth:`intercepted_unops`)
+        this function is executed instead of the builtin operator.  This can
+        be used to fine tune the behavior of certain operators.
+
+        .. versionadded:: 2.6
+        """
+        return self.unop_table[operator](arg)
+
+    def getitem(self, obj, argument):
+        """Subscribe an object from sandboxed code."""
+        try:
+            return obj[argument]
+        except (TypeError, LookupError):
+            if isinstance(argument, string_types):
+                try:
+                    attr = str(argument)
+                except Exception:
+                    pass
+                else:
+                    try:
+                        value = getattr(obj, attr)
+                    except AttributeError:
+                        pass
+                    else:
+                        if self.is_safe_attribute(obj, argument, value):
+                            return value
+                        return self.unsafe_undefined(obj, argument)
+        return self.undefined(obj=obj, name=argument)
+
+    def getattr(self, obj, attribute):
+        """Subscribe an object from sandboxed code and prefer the
+        attribute.  The attribute passed *must* be a bytestring.
+        """
+        try:
+            value = getattr(obj, attribute)
+        except AttributeError:
+            try:
+                return obj[attribute]
+            except (TypeError, LookupError):
+                pass
+        else:
+            if self.is_safe_attribute(obj, attribute, value):
+                return value
+            return self.unsafe_undefined(obj, attribute)
+        return self.undefined(obj=obj, name=attribute)
+
+    def unsafe_undefined(self, obj, attribute):
+        """Return an undefined object for unsafe attributes."""
+        return self.undefined('access to attribute %r of %r '
+                              'object is unsafe.' % (
+            attribute,
+            obj.__class__.__name__
+        ), name=attribute, obj=obj, exc=SecurityError)
+
+    def format_string(self, s, args, kwargs, format_func=None):
+        """If a format call is detected, then this is routed through this
+        method so that our safety sandbox can be used for it.
+        """
+        if isinstance(s, Markup):
+            formatter = SandboxedEscapeFormatter(self, s.escape)
+        else:
+            formatter = SandboxedFormatter(self)
+
+        if format_func is not None and format_func.__name__ == 'format_map':
+            if len(args) != 1 or kwargs:
+                raise TypeError(
+                    'format_map() takes exactly one argument %d given'
+                    % (len(args) + (kwargs is not None))
+                )
+
+            kwargs = args[0]
+            args = None
+
+        kwargs = _MagicFormatMapping(args, kwargs)
+        rv = formatter.vformat(s, args, kwargs)
+        return type(s)(rv)
+
+    def call(__self, __context, __obj, *args, **kwargs):
+        """Call an object from sandboxed code."""
+        fmt = inspect_format_method(__obj)
+        if fmt is not None:
+            return __self.format_string(fmt, args, kwargs, __obj)
+
+        # the double prefixes are to avoid double keyword argument
+        # errors when proxying the call.
+        if not __self.is_safe_callable(__obj):
+            raise SecurityError('%r is not safely callable' % (__obj,))
+        return __context.call(__obj, *args, **kwargs)
+
+
+class ImmutableSandboxedEnvironment(SandboxedEnvironment):
+    """Works exactly like the regular `SandboxedEnvironment` but does not
+    permit modifications on the builtin mutable objects `list`, `set`, and
+    `dict` by using the :func:`modifies_known_mutable` function.
+    """
+
+    def is_safe_attribute(self, obj, attr, value):
+        if not SandboxedEnvironment.is_safe_attribute(self, obj, attr, value):
+            return False
+        return not modifies_known_mutable(obj, attr)
+
+
+# This really is not a public API apparenlty.
+try:
+    from _string import formatter_field_name_split
+except ImportError:
+    def formatter_field_name_split(field_name):
+        return field_name._formatter_field_name_split()
+
+
+class SandboxedFormatterMixin(object):
+
+    def __init__(self, env):
+        self._env = env
+
+    def get_field(self, field_name, args, kwargs):
+        first, rest = formatter_field_name_split(field_name)
+        obj = self.get_value(first, args, kwargs)
+        for is_attr, i in rest:
+            if is_attr:
+                obj = self._env.getattr(obj, i)
+            else:
+                obj = self._env.getitem(obj, i)
+        return obj, first
+
+class SandboxedFormatter(SandboxedFormatterMixin, Formatter):
+
+    def __init__(self, env):
+        SandboxedFormatterMixin.__init__(self, env)
+        Formatter.__init__(self)
+
+class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter):
+
+    def __init__(self, env, escape):
+        SandboxedFormatterMixin.__init__(self, env)
+        EscapeFormatter.__init__(self, escape)

python/jinja2/tests.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.tests
+    ~~~~~~~~~~~~
+
+    Jinja test functions. Used with the "is" operator.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import operator
+import re
+from collections import Mapping
+from jinja2.runtime import Undefined
+from jinja2._compat import text_type, string_types, integer_types
+import decimal
+
+number_re = re.compile(r'^-?\d+(\.\d+)?$')
+regex_type = type(number_re)
+
+
+test_callable = callable
+
+
+def test_odd(value):
+    """Return true if the variable is odd."""
+    return value % 2 == 1
+
+
+def test_even(value):
+    """Return true if the variable is even."""
+    return value % 2 == 0
+
+
+def test_divisibleby(value, num):
+    """Check if a variable is divisible by a number."""
+    return value % num == 0
+
+
+def test_defined(value):
+    """Return true if the variable is defined:
+
+    .. sourcecode:: jinja
+
+        {% if variable is defined %}
+            value of variable: {{ variable }}
+        {% else %}
+            variable is not defined
+        {% endif %}
+
+    See the :func:`default` filter for a simple way to set undefined
+    variables.
+    """
+    return not isinstance(value, Undefined)
+
+
+def test_undefined(value):
+    """Like :func:`defined` but the other way round."""
+    return isinstance(value, Undefined)
+
+
+def test_none(value):
+    """Return true if the variable is none."""
+    return value is None
+
+
+def test_lower(value):
+    """Return true if the variable is lowercased."""
+    return text_type(value).islower()
+
+
+def test_upper(value):
+    """Return true if the variable is uppercased."""
+    return text_type(value).isupper()
+
+
+def test_string(value):
+    """Return true if the object is a string."""
+    return isinstance(value, string_types)
+
+
+def test_mapping(value):
+    """Return true if the object is a mapping (dict etc.).
+
+    .. versionadded:: 2.6
+    """
+    return isinstance(value, Mapping)
+
+
+def test_number(value):
+    """Return true if the variable is a number."""
+    return isinstance(value, integer_types + (float, complex, decimal.Decimal))
+
+
+def test_sequence(value):
+    """Return true if the variable is a sequence. Sequences are variables
+    that are iterable.
+    """
+    try:
+        len(value)
+        value.__getitem__
+    except:
+        return False
+    return True
+
+
+def test_sameas(value, other):
+    """Check if an object points to the same memory address than another
+    object:
+
+    .. sourcecode:: jinja
+
+        {% if foo.attribute is sameas false %}
+            the foo attribute really is the `False` singleton
+        {% endif %}
+    """
+    return value is other
+
+
+def test_iterable(value):
+    """Check if it's possible to iterate over an object."""
+    try:
+        iter(value)
+    except TypeError:
+        return False
+    return True
+
+
+def test_escaped(value):
+    """Check if the value is escaped."""
+    return hasattr(value, '__html__')
+
+
+def test_in(value, seq):
+    """Check if value is in seq.
+
+    .. versionadded:: 2.10
+    """
+    return value in seq
+
+
+TESTS = {
+    'odd':              test_odd,
+    'even':             test_even,
+    'divisibleby':      test_divisibleby,
+    'defined':          test_defined,
+    'undefined':        test_undefined,
+    'none':             test_none,
+    'lower':            test_lower,
+    'upper':            test_upper,
+    'string':           test_string,
+    'mapping':          test_mapping,
+    'number':           test_number,
+    'sequence':         test_sequence,
+    'iterable':         test_iterable,
+    'callable':         test_callable,
+    'sameas':           test_sameas,
+    'escaped':          test_escaped,
+    'in':               test_in,
+    '==':               operator.eq,
+    'eq':               operator.eq,
+    'equalto':          operator.eq,
+    '!=':               operator.ne,
+    'ne':               operator.ne,
+    '>':                operator.gt,
+    'gt':               operator.gt,
+    'greaterthan':      operator.gt,
+    'ge':               operator.ge,
+    '>=':               operator.ge,
+    '<':                operator.lt,
+    'lt':               operator.lt,
+    'lessthan':         operator.lt,
+    '<=':               operator.le,
+    'le':               operator.le,
+}

python/jinja2/utils.py

@@ -0,0 +1,647 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.utils
+    ~~~~~~~~~~~~
+
+    Utility functions.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+import json
+import errno
+from collections import deque
+from threading import Lock
+from jinja2._compat import text_type, string_types, implements_iterator, \
+     url_quote
+
+
+_word_split_re = re.compile(r'(\s+)')
+_punctuation_re = re.compile(
+    '^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % (
+        '|'.join(map(re.escape, ('(', '<', '&lt;'))),
+        '|'.join(map(re.escape, ('.', ',', ')', '>', '\n', '&gt;')))
+    )
+)
+_simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$')
+_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
+_entity_re = re.compile(r'&([^;]+);')
+_letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+_digits = '0123456789'
+
+# special singleton representing missing values for the runtime
+missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})()
+
+# internal code
+internal_code = set()
+
+concat = u''.join
+
+_slash_escape = '\\/' not in json.dumps('/')
+
+
+def contextfunction(f):
+    """This decorator can be used to mark a function or method context callable.
+    A context callable is passed the active :class:`Context` as first argument when
+    called from the template.  This is useful if a function wants to get access
+    to the context or functions provided on the context object.  For example
+    a function that returns a sorted list of template variables the current
+    template exports could look like this::
+
+        @contextfunction
+        def get_exported_names(context):
+            return sorted(context.exported_vars)
+    """
+    f.contextfunction = True
+    return f
+
+
+def evalcontextfunction(f):
+    """This decorator can be used to mark a function or method as an eval
+    context callable.  This is similar to the :func:`contextfunction`
+    but instead of passing the context, an evaluation context object is
+    passed.  For more information about the eval context, see
+    :ref:`eval-context`.
+
+    .. versionadded:: 2.4
+    """
+    f.evalcontextfunction = True
+    return f
+
+
+def environmentfunction(f):
+    """This decorator can be used to mark a function or method as environment
+    callable.  This decorator works exactly like the :func:`contextfunction`
+    decorator just that the first argument is the active :class:`Environment`
+    and not context.
+    """
+    f.environmentfunction = True
+    return f
+
+
+def internalcode(f):
+    """Marks the function as internally used"""
+    internal_code.add(f.__code__)
+    return f
+
+
+def is_undefined(obj):
+    """Check if the object passed is undefined.  This does nothing more than
+    performing an instance check against :class:`Undefined` but looks nicer.
+    This can be used for custom filters or tests that want to react to
+    undefined variables.  For example a custom default filter can look like
+    this::
+
+        def default(var, default=''):
+            if is_undefined(var):
+                return default
+            return var
+    """
+    from jinja2.runtime import Undefined
+    return isinstance(obj, Undefined)
+
+
+def consume(iterable):
+    """Consumes an iterable without doing anything with it."""
+    for event in iterable:
+        pass
+
+
+def clear_caches():
+    """Jinja2 keeps internal caches for environments and lexers.  These are
+    used so that Jinja2 doesn't have to recreate environments and lexers all
+    the time.  Normally you don't have to care about that but if you are
+    measuring memory consumption you may want to clean the caches.
+    """
+    from jinja2.environment import _spontaneous_environments
+    from jinja2.lexer import _lexer_cache
+    _spontaneous_environments.clear()
+    _lexer_cache.clear()
+
+
+def import_string(import_name, silent=False):
+    """Imports an object based on a string.  This is useful if you want to
+    use import paths as endpoints or something similar.  An import path can
+    be specified either in dotted notation (``xml.sax.saxutils.escape``)
+    or with a colon as object delimiter (``xml.sax.saxutils:escape``).
+
+    If the `silent` is True the return value will be `None` if the import
+    fails.
+
+    :return: imported object
+    """
+    try:
+        if ':' in import_name:
+            module, obj = import_name.split(':', 1)
+        elif '.' in import_name:
+            items = import_name.split('.')
+            module = '.'.join(items[:-1])
+            obj = items[-1]
+        else:
+            return __import__(import_name)
+        return getattr(__import__(module, None, None, [obj]), obj)
+    except (ImportError, AttributeError):
+        if not silent:
+            raise
+
+
+def open_if_exists(filename, mode='rb'):
+    """Returns a file descriptor for the filename if that file exists,
+    otherwise `None`.
+    """
+    try:
+        return open(filename, mode)
+    except IOError as e:
+        if e.errno not in (errno.ENOENT, errno.EISDIR, errno.EINVAL):
+            raise
+
+
+def object_type_repr(obj):
+    """Returns the name of the object's type.  For some recognized
+    singletons the name of the object is returned instead. (For
+    example for `None` and `Ellipsis`).
+    """
+    if obj is None:
+        return 'None'
+    elif obj is Ellipsis:
+        return 'Ellipsis'
+    # __builtin__ in 2.x, builtins in 3.x
+    if obj.__class__.__module__ in ('__builtin__', 'builtins'):
+        name = obj.__class__.__name__
+    else:
+        name = obj.__class__.__module__ + '.' + obj.__class__.__name__
+    return '%s object' % name
+
+
+def pformat(obj, verbose=False):
+    """Prettyprint an object.  Either use the `pretty` library or the
+    builtin `pprint`.
+    """
+    try:
+        from pretty import pretty
+        return pretty(obj, verbose=verbose)
+    except ImportError:
+        from pprint import pformat
+        return pformat(obj)
+
+
+def urlize(text, trim_url_limit=None, rel=None, target=None):
+    """Converts any URLs in text into clickable links. Works on http://,
+    https:// and www. links. Links can have trailing punctuation (periods,
+    commas, close-parens) and leading punctuation (opening parens) and
+    it'll still do the right thing.
+
+    If trim_url_limit is not None, the URLs in link text will be limited
+    to trim_url_limit characters.
+
+    If nofollow is True, the URLs in link text will get a rel="nofollow"
+    attribute.
+
+    If target is not None, a target attribute will be added to the link.
+    """
+    trim_url = lambda x, limit=trim_url_limit: limit is not None \
+                         and (x[:limit] + (len(x) >=limit and '...'
+                         or '')) or x
+    words = _word_split_re.split(text_type(escape(text)))
+    rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or ''
+    target_attr = target and ' target="%s"' % escape(target) or ''
+
+    for i, word in enumerate(words):
+        match = _punctuation_re.match(word)
+        if match:
+            lead, middle, trail = match.groups()
+            if middle.startswith('www.') or (
+                '@' not in middle and
+                not middle.startswith('http://') and
+                not middle.startswith('https://') and
+                len(middle) > 0 and
+                middle[0] in _letters + _digits and (
+                    middle.endswith('.org') or
+                    middle.endswith('.net') or
+                    middle.endswith('.com')
+                )):
+                middle = '<a href="http://%s"%s%s>%s</a>' % (middle,
+                    rel_attr, target_attr, trim_url(middle))
+            if middle.startswith('http://') or \
+               middle.startswith('https://'):
+                middle = '<a href="%s"%s%s>%s</a>' % (middle,
+                    rel_attr, target_attr, trim_url(middle))
+            if '@' in middle and not middle.startswith('www.') and \
+               not ':' in middle and _simple_email_re.match(middle):
+                middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
+            if lead + middle + trail != word:
+                words[i] = lead + middle + trail
+    return u''.join(words)
+
+
+def generate_lorem_ipsum(n=5, html=True, min=20, max=100):
+    """Generate some lorem ipsum for the template."""
+    from jinja2.constants import LOREM_IPSUM_WORDS
+    from random import choice, randrange
+    words = LOREM_IPSUM_WORDS.split()
+    result = []
+
+    for _ in range(n):
+        next_capitalized = True
+        last_comma = last_fullstop = 0
+        word = None
+        last = None
+        p = []
+
+        # each paragraph contains out of 20 to 100 words.
+        for idx, _ in enumerate(range(randrange(min, max))):
+            while True:
+                word = choice(words)
+                if word != last:
+                    last = word
+                    break
+            if next_capitalized:
+                word = word.capitalize()
+                next_capitalized = False
+            # add commas
+            if idx - randrange(3, 8) > last_comma:
+                last_comma = idx
+                last_fullstop += 2
+                word += ','
+            # add end of sentences
+            if idx - randrange(10, 20) > last_fullstop:
+                last_comma = last_fullstop = idx
+                word += '.'
+                next_capitalized = True
+            p.append(word)
+
+        # ensure that the paragraph ends with a dot.
+        p = u' '.join(p)
+        if p.endswith(','):
+            p = p[:-1] + '.'
+        elif not p.endswith('.'):
+            p += '.'
+        result.append(p)
+
+    if not html:
+        return u'\n\n'.join(result)
+    return Markup(u'\n'.join(u'<p>%s</p>' % escape(x) for x in result))
+
+
+def unicode_urlencode(obj, charset='utf-8', for_qs=False):
+    """URL escapes a single bytestring or unicode string with the
+    given charset if applicable to URL safe quoting under all rules
+    that need to be considered under all supported Python versions.
+
+    If non strings are provided they are converted to their unicode
+    representation first.
+    """
+    if not isinstance(obj, string_types):
+        obj = text_type(obj)
+    if isinstance(obj, text_type):
+        obj = obj.encode(charset)
+    safe = not for_qs and b'/' or b''
+    rv = text_type(url_quote(obj, safe))
+    if for_qs:
+        rv = rv.replace('%20', '+')
+    return rv
+
+
+class LRUCache(object):
+    """A simple LRU Cache implementation."""
+
+    # this is fast for small capacities (something below 1000) but doesn't
+    # scale.  But as long as it's only used as storage for templates this
+    # won't do any harm.
+
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self._mapping = {}
+        self._queue = deque()
+        self._postinit()
+
+    def _postinit(self):
+        # alias all queue methods for faster lookup
+        self._popleft = self._queue.popleft
+        self._pop = self._queue.pop
+        self._remove = self._queue.remove
+        self._wlock = Lock()
+        self._append = self._queue.append
+
+    def __getstate__(self):
+        return {
+            'capacity':     self.capacity,
+            '_mapping':     self._mapping,
+            '_queue':       self._queue
+        }
+
+    def __setstate__(self, d):
+        self.__dict__.update(d)
+        self._postinit()
+
+    def __getnewargs__(self):
+        return (self.capacity,)
+
+    def copy(self):
+        """Return a shallow copy of the instance."""
+        rv = self.__class__(self.capacity)
+        rv._mapping.update(self._mapping)
+        rv._queue = deque(self._queue)
+        return rv
+
+    def get(self, key, default=None):
+        """Return an item from the cache dict or `default`"""
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def setdefault(self, key, default=None):
+        """Set `default` if the key is not in the cache otherwise
+        leave unchanged. Return the value of this key.
+        """
+        self._wlock.acquire()
+        try:
+            try:
+                return self[key]
+            except KeyError:
+                self[key] = default
+                return default
+        finally:
+            self._wlock.release()
+
+    def clear(self):
+        """Clear the cache."""
+        self._wlock.acquire()
+        try:
+            self._mapping.clear()
+            self._queue.clear()
+        finally:
+            self._wlock.release()
+
+    def __contains__(self, key):
+        """Check if a key exists in this cache."""
+        return key in self._mapping
+
+    def __len__(self):
+        """Return the current size of the cache."""
+        return len(self._mapping)
+
+    def __repr__(self):
+        return '<%s %r>' % (
+            self.__class__.__name__,
+            self._mapping
+        )
+
+    def __getitem__(self, key):
+        """Get an item from the cache. Moves the item up so that it has the
+        highest priority then.
+
+        Raise a `KeyError` if it does not exist.
+        """
+        self._wlock.acquire()
+        try:
+            rv = self._mapping[key]
+            if self._queue[-1] != key:
+                try:
+                    self._remove(key)
+                except ValueError:
+                    # if something removed the key from the container
+                    # when we read, ignore the ValueError that we would
+                    # get otherwise.
+                    pass
+                self._append(key)
+            return rv
+        finally:
+            self._wlock.release()
+
+    def __setitem__(self, key, value):
+        """Sets the value for an item. Moves the item up so that it
+        has the highest priority then.
+        """
+        self._wlock.acquire()
+        try:
+            if key in self._mapping:
+                self._remove(key)
+            elif len(self._mapping) == self.capacity:
+                del self._mapping[self._popleft()]
+            self._append(key)
+            self._mapping[key] = value
+        finally:
+            self._wlock.release()
+
+    def __delitem__(self, key):
+        """Remove an item from the cache dict.
+        Raise a `KeyError` if it does not exist.
+        """
+        self._wlock.acquire()
+        try:
+            del self._mapping[key]
+            try:
+                self._remove(key)
+            except ValueError:
+                # __getitem__ is not locked, it might happen
+                pass
+        finally:
+            self._wlock.release()
+
+    def items(self):
+        """Return a list of items."""
+        result = [(key, self._mapping[key]) for key in list(self._queue)]
+        result.reverse()
+        return result
+
+    def iteritems(self):
+        """Iterate over all items."""
+        return iter(self.items())
+
+    def values(self):
+        """Return a list of all values."""
+        return [x[1] for x in self.items()]
+
+    def itervalue(self):
+        """Iterate over all values."""
+        return iter(self.values())
+
+    def keys(self):
+        """Return a list of all keys ordered by most recent usage."""
+        return list(self)
+
+    def iterkeys(self):
+        """Iterate over all keys in the cache dict, ordered by
+        the most recent usage.
+        """
+        return reversed(tuple(self._queue))
+
+    __iter__ = iterkeys
+
+    def __reversed__(self):
+        """Iterate over the values in the cache dict, oldest items
+        coming first.
+        """
+        return iter(tuple(self._queue))
+
+    __copy__ = copy
+
+
+# register the LRU cache as mutable mapping if possible
+try:
+    from collections import MutableMapping
+    MutableMapping.register(LRUCache)
+except ImportError:
+    pass
+
+
+def select_autoescape(enabled_extensions=('html', 'htm', 'xml'),
+                      disabled_extensions=(),
+                      default_for_string=True,
+                      default=False):
+    """Intelligently sets the initial value of autoescaping based on the
+    filename of the template.  This is the recommended way to configure
+    autoescaping if you do not want to write a custom function yourself.
+
+    If you want to enable it for all templates created from strings or
+    for all templates with `.html` and `.xml` extensions::
+
+        from jinja2 import Environment, select_autoescape
+        env = Environment(autoescape=select_autoescape(
+            enabled_extensions=('html', 'xml'),
+            default_for_string=True,
+        ))
+
+    Example configuration to turn it on at all times except if the template
+    ends with `.txt`::
+
+        from jinja2 import Environment, select_autoescape
+        env = Environment(autoescape=select_autoescape(
+            disabled_extensions=('txt',),
+            default_for_string=True,
+            default=True,
+        ))
+
+    The `enabled_extensions` is an iterable of all the extensions that
+    autoescaping should be enabled for.  Likewise `disabled_extensions` is
+    a list of all templates it should be disabled for.  If a template is
+    loaded from a string then the default from `default_for_string` is used.
+    If nothing matches then the initial value of autoescaping is set to the
+    value of `default`.
+
+    For security reasons this function operates case insensitive.
+
+    .. versionadded:: 2.9
+    """
+    enabled_patterns = tuple('.' + x.lstrip('.').lower()
+                             for x in enabled_extensions)
+    disabled_patterns = tuple('.' + x.lstrip('.').lower()
+                              for x in disabled_extensions)
+    def autoescape(template_name):
+        if template_name is None:
+            return default_for_string
+        template_name = template_name.lower()
+        if template_name.endswith(enabled_patterns):
+            return True
+        if template_name.endswith(disabled_patterns):
+            return False
+        return default
+    return autoescape
+
+
+def htmlsafe_json_dumps(obj, dumper=None, **kwargs):
+    """Works exactly like :func:`dumps` but is safe for use in ``<script>``
+    tags.  It accepts the same arguments and returns a JSON string.  Note that
+    this is available in templates through the ``|tojson`` filter which will
+    also mark the result as safe.  Due to how this function escapes certain
+    characters this is safe even if used outside of ``<script>`` tags.
+
+    The following characters are escaped in strings:
+
+    -   ``<``
+    -   ``>``
+    -   ``&``
+    -   ``'``
+
+    This makes it safe to embed such strings in any place in HTML with the
+    notable exception of double quoted attributes.  In that case single
+    quote your attributes or HTML escape it in addition.
+    """
+    if dumper is None:
+        dumper = json.dumps
+    rv = dumper(obj, **kwargs) \
+        .replace(u'<', u'\\u003c') \
+        .replace(u'>', u'\\u003e') \
+        .replace(u'&', u'\\u0026') \
+        .replace(u"'", u'\\u0027')
+    return Markup(rv)
+
+
+@implements_iterator
+class Cycler(object):
+    """A cycle helper for templates."""
+
+    def __init__(self, *items):
+        if not items:
+            raise RuntimeError('at least one item has to be provided')
+        self.items = items
+        self.reset()
+
+    def reset(self):
+        """Resets the cycle."""
+        self.pos = 0
+
+    @property
+    def current(self):
+        """Returns the current item."""
+        return self.items[self.pos]
+
+    def next(self):
+        """Goes one item ahead and returns it."""
+        rv = self.current
+        self.pos = (self.pos + 1) % len(self.items)
+        return rv
+
+    __next__ = next
+
+
+class Joiner(object):
+    """A joining helper for templates."""
+
+    def __init__(self, sep=u', '):
+        self.sep = sep
+        self.used = False
+
+    def __call__(self):
+        if not self.used:
+            self.used = True
+            return u''
+        return self.sep
+
+
+class Namespace(object):
+    """A namespace object that can hold arbitrary attributes.  It may be
+    initialized from a dictionary or with keyword argments."""
+
+    def __init__(*args, **kwargs):
+        self, args = args[0], args[1:]
+        self.__attrs = dict(*args, **kwargs)
+
+    def __getattribute__(self, name):
+        if name == '_Namespace__attrs':
+            return object.__getattribute__(self, name)
+        try:
+            return self.__attrs[name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __setitem__(self, name, value):
+        self.__attrs[name] = value
+
+    def __repr__(self):
+        return '<Namespace %r>' % self.__attrs
+
+
+# does this python version support async for in and async generators?
+try:
+    exec('async def _():\n async for _ in ():\n  yield _')
+    have_async_gen = True
+except SyntaxError:
+    have_async_gen = False
+
+
+# Imported here because that's where it was in the past
+from markupsafe import Markup, escape, soft_unicode

python/jinja2/visitor.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.visitor
+    ~~~~~~~~~~~~~~
+
+    This module implements a visitor for the nodes.
+
+    :copyright: (c) 2017 by the Jinja Team.
+    :license: BSD.
+"""
+from jinja2.nodes import Node
+
+
+class NodeVisitor(object):
+    """Walks the abstract syntax tree and call visitor functions for every
+    node found.  The visitor functions may return values which will be
+    forwarded by the `visit` method.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `get_visitor` function.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+    """
+
+    def get_visitor(self, node):
+        """Return the visitor function for this node or `None` if no visitor
+        exists for this node.  In that case the generic visit function is
+        used instead.
+        """
+        method = 'visit_' + node.__class__.__name__
+        return getattr(self, method, None)
+
+    def visit(self, node, *args, **kwargs):
+        """Visit a node."""
+        f = self.get_visitor(node)
+        if f is not None:
+            return f(node, *args, **kwargs)
+        return self.generic_visit(node, *args, **kwargs)
+
+    def generic_visit(self, node, *args, **kwargs):
+        """Called if no explicit visitor function exists for a node."""
+        for node in node.iter_child_nodes():
+            self.visit(node, *args, **kwargs)
+
+
+class NodeTransformer(NodeVisitor):
+    """Walks the abstract syntax tree and allows modifications of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor functions to replace or remove the old node.  If the return
+    value of the visitor function is `None` the node will be removed
+    from the previous location otherwise it's replaced with the return
+    value.  The return value may be the original node in which case no
+    replacement takes place.
+    """
+
+    def generic_visit(self, node, *args, **kwargs):
+        for field, old_value in node.iter_fields():
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, Node):
+                        value = self.visit(value, *args, **kwargs)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, Node):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, Node):
+                new_node = self.visit(old_value, *args, **kwargs)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
+
+    def visit_list(self, node, *args, **kwargs):
+        """As transformers may return lists in some places this method
+        can be used to enforce a list as return value.
+        """
+        rv = self.visit(node, *args, **kwargs)
+        if not isinstance(rv, list):
+            rv = [rv]
+        return rv

python/markupsafe/__init__.py

@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+"""
+    markupsafe
+    ~~~~~~~~~~
+
+    Implements a Markup string.
+
+    :copyright: (c) 2010 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+import string
+from collections import Mapping
+from markupsafe._compat import text_type, string_types, int_types, \
+     unichr, iteritems, PY2
+
+__version__ = "1.0"
+
+__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent']
+
+
+_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
+_entity_re = re.compile(r'&([^& ;]+);')
+
+
+class Markup(text_type):
+    r"""Marks a string as being safe for inclusion in HTML/XML output without
+    needing to be escaped.  This implements the `__html__` interface a couple
+    of frameworks and web applications use.  :class:`Markup` is a direct
+    subclass of `unicode` and provides all the methods of `unicode` just that
+    it escapes arguments passed and always returns `Markup`.
+
+    The `escape` function returns markup objects so that double escaping can't
+    happen.
+
+    The constructor of the :class:`Markup` class can be used for three
+    different things:  When passed an unicode object it's assumed to be safe,
+    when passed an object with an HTML representation (has an `__html__`
+    method) that representation is used, otherwise the object passed is
+    converted into a unicode string and then assumed to be safe:
+
+    >>> Markup("Hello <em>World</em>!")
+    Markup(u'Hello <em>World</em>!')
+    >>> class Foo(object):
+    ...  def __html__(self):
+    ...   return '<a href="#">foo</a>'
+    ...
+    >>> Markup(Foo())
+    Markup(u'<a href="#">foo</a>')
+
+    If you want object passed being always treated as unsafe you can use the
+    :meth:`escape` classmethod to create a :class:`Markup` object:
+
+    >>> Markup.escape("Hello <em>World</em>!")
+    Markup(u'Hello &lt;em&gt;World&lt;/em&gt;!')
+
+    Operations on a markup string are markup aware which means that all
+    arguments are passed through the :func:`escape` function:
+
+    >>> em = Markup("<em>%s</em>")
+    >>> em % "foo & bar"
+    Markup(u'<em>foo &amp; bar</em>')
+    >>> strong = Markup("<strong>%(text)s</strong>")
+    >>> strong % {'text': '<blink>hacker here</blink>'}
+    Markup(u'<strong>&lt;blink&gt;hacker here&lt;/blink&gt;</strong>')
+    >>> Markup("<em>Hello</em> ") + "<foo>"
+    Markup(u'<em>Hello</em> &lt;foo&gt;')
+    """
+    __slots__ = ()
+
+    def __new__(cls, base=u'', encoding=None, errors='strict'):
+        if hasattr(base, '__html__'):
+            base = base.__html__()
+        if encoding is None:
+            return text_type.__new__(cls, base)
+        return text_type.__new__(cls, base, encoding, errors)
+
+    def __html__(self):
+        return self
+
+    def __add__(self, other):
+        if isinstance(other, string_types) or hasattr(other, '__html__'):
+            return self.__class__(super(Markup, self).__add__(self.escape(other)))
+        return NotImplemented
+
+    def __radd__(self, other):
+        if hasattr(other, '__html__') or isinstance(other, string_types):
+            return self.escape(other).__add__(self)
+        return NotImplemented
+
+    def __mul__(self, num):
+        if isinstance(num, int_types):
+            return self.__class__(text_type.__mul__(self, num))
+        return NotImplemented
+    __rmul__ = __mul__
+
+    def __mod__(self, arg):
+        if isinstance(arg, tuple):
+            arg = tuple(_MarkupEscapeHelper(x, self.escape) for x in arg)
+        else:
+            arg = _MarkupEscapeHelper(arg, self.escape)
+        return self.__class__(text_type.__mod__(self, arg))
+
+    def __repr__(self):
+        return '%s(%s)' % (
+            self.__class__.__name__,
+            text_type.__repr__(self)
+        )
+
+    def join(self, seq):
+        return self.__class__(text_type.join(self, map(self.escape, seq)))
+    join.__doc__ = text_type.join.__doc__
+
+    def split(self, *args, **kwargs):
+        return list(map(self.__class__, text_type.split(self, *args, **kwargs)))
+    split.__doc__ = text_type.split.__doc__
+
+    def rsplit(self, *args, **kwargs):
+        return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs)))
+    rsplit.__doc__ = text_type.rsplit.__doc__
+
+    def splitlines(self, *args, **kwargs):
+        return list(map(self.__class__, text_type.splitlines(
+            self, *args, **kwargs)))
+    splitlines.__doc__ = text_type.splitlines.__doc__
+
+    def unescape(self):
+        r"""Unescape markup again into an text_type string.  This also resolves
+        known HTML4 and XHTML entities:
+
+        >>> Markup("Main &raquo; <em>About</em>").unescape()
+        u'Main \xbb <em>About</em>'
+        """
+        from markupsafe._constants import HTML_ENTITIES
+        def handle_match(m):
+            name = m.group(1)
+            if name in HTML_ENTITIES:
+                return unichr(HTML_ENTITIES[name])
+            try:
+                if name[:2] in ('#x', '#X'):
+                    return unichr(int(name[2:], 16))
+                elif name.startswith('#'):
+                    return unichr(int(name[1:]))
+            except ValueError:
+                pass
+            # Don't modify unexpected input.
+            return m.group()
+        return _entity_re.sub(handle_match, text_type(self))
+
+    def striptags(self):
+        r"""Unescape markup into an text_type string and strip all tags.  This
+        also resolves known HTML4 and XHTML entities.  Whitespace is
+        normalized to one:
+
+        >>> Markup("Main &raquo;  <em>About</em>").striptags()
+        u'Main \xbb About'
+        """
+        stripped = u' '.join(_striptags_re.sub('', self).split())
+        return Markup(stripped).unescape()
+
+    @classmethod
+    def escape(cls, s):
+        """Escape the string.  Works like :func:`escape` with the difference
+        that for subclasses of :class:`Markup` this function would return the
+        correct subclass.
+        """
+        rv = escape(s)
+        if rv.__class__ is not cls:
+            return cls(rv)
+        return rv
+
+    def make_simple_escaping_wrapper(name):
+        orig = getattr(text_type, name)
+        def func(self, *args, **kwargs):
+            args = _escape_argspec(list(args), enumerate(args), self.escape)
+            _escape_argspec(kwargs, iteritems(kwargs), self.escape)
+            return self.__class__(orig(self, *args, **kwargs))
+        func.__name__ = orig.__name__
+        func.__doc__ = orig.__doc__
+        return func
+
+    for method in '__getitem__', 'capitalize', \
+                  'title', 'lower', 'upper', 'replace', 'ljust', \
+                  'rjust', 'lstrip', 'rstrip', 'center', 'strip', \
+                  'translate', 'expandtabs', 'swapcase', 'zfill':
+        locals()[method] = make_simple_escaping_wrapper(method)
+
+    # new in python 2.5
+    if hasattr(text_type, 'partition'):
+        def partition(self, sep):
+            return tuple(map(self.__class__,
+                             text_type.partition(self, self.escape(sep))))
+        def rpartition(self, sep):
+            return tuple(map(self.__class__,
+                             text_type.rpartition(self, self.escape(sep))))
+
+    # new in python 2.6
+    if hasattr(text_type, 'format'):
+        def format(*args, **kwargs):
+            self, args = args[0], args[1:]
+            formatter = EscapeFormatter(self.escape)
+            kwargs = _MagicFormatMapping(args, kwargs)
+            return self.__class__(formatter.vformat(self, args, kwargs))
+
+        def __html_format__(self, format_spec):
+            if format_spec:
+                raise ValueError('Unsupported format specification '
+                                 'for Markup.')
+            return self
+
+    # not in python 3
+    if hasattr(text_type, '__getslice__'):
+        __getslice__ = make_simple_escaping_wrapper('__getslice__')
+
+    del method, make_simple_escaping_wrapper
+
+
+class _MagicFormatMapping(Mapping):
+    """This class implements a dummy wrapper to fix a bug in the Python
+    standard library for string formatting.
+
+    See http://bugs.python.org/issue13598 for information about why
+    this is necessary.
+    """
+
+    def __init__(self, args, kwargs):
+        self._args = args
+        self._kwargs = kwargs
+        self._last_index = 0
+
+    def __getitem__(self, key):
+        if key == '':
+            idx = self._last_index
+            self._last_index += 1
+            try:
+                return self._args[idx]
+            except LookupError:
+                pass
+            key = str(idx)
+        return self._kwargs[key]
+
+    def __iter__(self):
+        return iter(self._kwargs)
+
+    def __len__(self):
+        return len(self._kwargs)
+
+
+if hasattr(text_type, 'format'):
+    class EscapeFormatter(string.Formatter):
+
+        def __init__(self, escape):
+            self.escape = escape
+
+        def format_field(self, value, format_spec):
+            if hasattr(value, '__html_format__'):
+                rv = value.__html_format__(format_spec)
+            elif hasattr(value, '__html__'):
+                if format_spec:
+                    raise ValueError('No format specification allowed '
+                                     'when formatting an object with '
+                                     'its __html__ method.')
+                rv = value.__html__()
+            else:
+                # We need to make sure the format spec is unicode here as
+                # otherwise the wrong callback methods are invoked.  For
+                # instance a byte string there would invoke __str__ and
+                # not __unicode__.
+                rv = string.Formatter.format_field(
+                    self, value, text_type(format_spec))
+            return text_type(self.escape(rv))
+
+
+def _escape_argspec(obj, iterable, escape):
+    """Helper for various string-wrapped functions."""
+    for key, value in iterable:
+        if hasattr(value, '__html__') or isinstance(value, string_types):
+            obj[key] = escape(value)
+    return obj
+
+
+class _MarkupEscapeHelper(object):
+    """Helper for Markup.__mod__"""
+
+    def __init__(self, obj, escape):
+        self.obj = obj
+        self.escape = escape
+
+    __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape)
+    __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj))
+    __repr__ = lambda s: str(s.escape(repr(s.obj)))
+    __int__ = lambda s: int(s.obj)
+    __float__ = lambda s: float(s.obj)
+
+
+# we have to import it down here as the speedups and native
+# modules imports the markup type which is define above.
+try:
+    from markupsafe._speedups import escape, escape_silent, soft_unicode
+except ImportError:
+    from markupsafe._native import escape, escape_silent, soft_unicode
+
+if not PY2:
+    soft_str = soft_unicode
+    __all__.append('soft_str')

python/markupsafe/_compat.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+    markupsafe._compat
+    ~~~~~~~~~~~~~~~~~~
+
+    Compatibility module for different Python versions.
+
+    :copyright: (c) 2013 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+
+PY2 = sys.version_info[0] == 2
+
+if not PY2:
+    text_type = str
+    string_types = (str,)
+    unichr = chr
+    int_types = (int,)
+    iteritems = lambda x: iter(x.items())
+else:
+    text_type = unicode
+    string_types = (str, unicode)
+    unichr = unichr
+    int_types = (int, long)
+    iteritems = lambda x: x.iteritems()

python/markupsafe/_constants.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+"""
+    markupsafe._constants
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Highlevel implementation of the Markup string.
+
+    :copyright: (c) 2010 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+HTML_ENTITIES = {
+    'AElig': 198,
+    'Aacute': 193,
+    'Acirc': 194,
+    'Agrave': 192,
+    'Alpha': 913,
+    'Aring': 197,
+    'Atilde': 195,
+    'Auml': 196,
+    'Beta': 914,
+    'Ccedil': 199,
+    'Chi': 935,
+    'Dagger': 8225,
+    'Delta': 916,
+    'ETH': 208,
+    'Eacute': 201,
+    'Ecirc': 202,
+    'Egrave': 200,
+    'Epsilon': 917,
+    'Eta': 919,
+    'Euml': 203,
+    'Gamma': 915,
+    'Iacute': 205,
+    'Icirc': 206,
+    'Igrave': 204,
+    'Iota': 921,
+    'Iuml': 207,
+    'Kappa': 922,
+    'Lambda': 923,
+    'Mu': 924,
+    'Ntilde': 209,
+    'Nu': 925,
+    'OElig': 338,
+    'Oacute': 211,
+    'Ocirc': 212,
+    'Ograve': 210,
+    'Omega': 937,
+    'Omicron': 927,
+    'Oslash': 216,
+    'Otilde': 213,
+    'Ouml': 214,
+    'Phi': 934,
+    'Pi': 928,
+    'Prime': 8243,
+    'Psi': 936,
+    'Rho': 929,
+    'Scaron': 352,
+    'Sigma': 931,
+    'THORN': 222,
+    'Tau': 932,
+    'Theta': 920,
+    'Uacute': 218,
+    'Ucirc': 219,
+    'Ugrave': 217,
+    'Upsilon': 933,
+    'Uuml': 220,
+    'Xi': 926,
+    'Yacute': 221,
+    'Yuml': 376,
+    'Zeta': 918,
+    'aacute': 225,
+    'acirc': 226,
+    'acute': 180,
+    'aelig': 230,
+    'agrave': 224,
+    'alefsym': 8501,
+    'alpha': 945,
+    'amp': 38,
+    'and': 8743,
+    'ang': 8736,
+    'apos': 39,
+    'aring': 229,
+    'asymp': 8776,
+    'atilde': 227,
+    'auml': 228,
+    'bdquo': 8222,
+    'beta': 946,
+    'brvbar': 166,
+    'bull': 8226,
+    'cap': 8745,
+    'ccedil': 231,
+    'cedil': 184,
+    'cent': 162,
+    'chi': 967,
+    'circ': 710,
+    'clubs': 9827,
+    'cong': 8773,
+    'copy': 169,
+    'crarr': 8629,
+    'cup': 8746,
+    'curren': 164,
+    'dArr': 8659,
+    'dagger': 8224,
+    'darr': 8595,
+    'deg': 176,
+    'delta': 948,
+    'diams': 9830,
+    'divide': 247,
+    'eacute': 233,
+    'ecirc': 234,
+    'egrave': 232,
+    'empty': 8709,
+    'emsp': 8195,
+    'ensp': 8194,
+    'epsilon': 949,
+    'equiv': 8801,
+    'eta': 951,
+    'eth': 240,
+    'euml': 235,
+    'euro': 8364,
+    'exist': 8707,
+    'fnof': 402,
+    'forall': 8704,
+    'frac12': 189,
+    'frac14': 188,
+    'frac34': 190,
+    'frasl': 8260,
+    'gamma': 947,
+    'ge': 8805,
+    'gt': 62,
+    'hArr': 8660,
+    'harr': 8596,
+    'hearts': 9829,
+    'hellip': 8230,
+    'iacute': 237,
+    'icirc': 238,
+    'iexcl': 161,
+    'igrave': 236,
+    'image': 8465,
+    'infin': 8734,
+    'int': 8747,
+    'iota': 953,
+    'iquest': 191,
+    'isin': 8712,
+    'iuml': 239,
+    'kappa': 954,
+    'lArr': 8656,
+    'lambda': 955,
+    'lang': 9001,
+    'laquo': 171,
+    'larr': 8592,
+    'lceil': 8968,
+    'ldquo': 8220,
+    'le': 8804,
+    'lfloor': 8970,
+    'lowast': 8727,
+    'loz': 9674,
+    'lrm': 8206,
+    'lsaquo': 8249,
+    'lsquo': 8216,
+    'lt': 60,
+    'macr': 175,
+    'mdash': 8212,
+    'micro': 181,
+    'middot': 183,
+    'minus': 8722,
+    'mu': 956,
+    'nabla': 8711,
+    'nbsp': 160,
+    'ndash': 8211,
+    'ne': 8800,
+    'ni': 8715,
+    'not': 172,
+    'notin': 8713,
+    'nsub': 8836,
+    'ntilde': 241,
+    'nu': 957,
+    'oacute': 243,
+    'ocirc': 244,
+    'oelig': 339,
+    'ograve': 242,
+    'oline': 8254,
+    'omega': 969,
+    'omicron': 959,
+    'oplus': 8853,
+    'or': 8744,
+    'ordf': 170,
+    'ordm': 186,
+    'oslash': 248,
+    'otilde': 245,
+    'otimes': 8855,
+    'ouml': 246,
+    'para': 182,
+    'part': 8706,
+    'permil': 8240,
+    'perp': 8869,
+    'phi': 966,
+    'pi': 960,
+    'piv': 982,
+    'plusmn': 177,
+    'pound': 163,
+    'prime': 8242,
+    'prod': 8719,
+    'prop': 8733,
+    'psi': 968,
+    'quot': 34,
+    'rArr': 8658,
+    'radic': 8730,
+    'rang': 9002,
+    'raquo': 187,
+    'rarr': 8594,
+    'rceil': 8969,
+    'rdquo': 8221,
+    'real': 8476,
+    'reg': 174,
+    'rfloor': 8971,
+    'rho': 961,
+    'rlm': 8207,
+    'rsaquo': 8250,
+    'rsquo': 8217,
+    'sbquo': 8218,
+    'scaron': 353,
+    'sdot': 8901,
+    'sect': 167,
+    'shy': 173,
+    'sigma': 963,
+    'sigmaf': 962,
+    'sim': 8764,
+    'spades': 9824,
+    'sub': 8834,
+    'sube': 8838,
+    'sum': 8721,
+    'sup': 8835,
+    'sup1': 185,
+    'sup2': 178,
+    'sup3': 179,
+    'supe': 8839,
+    'szlig': 223,
+    'tau': 964,
+    'there4': 8756,
+    'theta': 952,
+    'thetasym': 977,
+    'thinsp': 8201,
+    'thorn': 254,
+    'tilde': 732,
+    'times': 215,
+    'trade': 8482,
+    'uArr': 8657,
+    'uacute': 250,
+    'uarr': 8593,
+    'ucirc': 251,
+    'ugrave': 249,
+    'uml': 168,
+    'upsih': 978,
+    'upsilon': 965,
+    'uuml': 252,
+    'weierp': 8472,
+    'xi': 958,
+    'yacute': 253,
+    'yen': 165,
+    'yuml': 255,
+    'zeta': 950,
+    'zwj': 8205,
+    'zwnj': 8204
+}

python/markupsafe/_native.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+    markupsafe._native
+    ~~~~~~~~~~~~~~~~~~
+
+    Native Python implementation the C module is not compiled.
+
+    :copyright: (c) 2010 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+from markupsafe import Markup
+from markupsafe._compat import text_type
+
+
+def escape(s):
+    """Convert the characters &, <, >, ' and " in string s to HTML-safe
+    sequences.  Use this if you need to display text that might contain
+    such characters in HTML.  Marks return value as markup string.
+    """
+    if hasattr(s, '__html__'):
+        return s.__html__()
+    return Markup(text_type(s)
+        .replace('&', '&amp;')
+        .replace('>', '&gt;')
+        .replace('<', '&lt;')
+        .replace("'", '&#39;')
+        .replace('"', '&#34;')
+    )
+
+
+def escape_silent(s):
+    """Like :func:`escape` but converts `None` into an empty
+    markup string.
+    """
+    if s is None:
+        return Markup()
+    return escape(s)
+
+
+def soft_unicode(s):
+    """Make a string unicode if it isn't already.  That way a markup
+    string is not converted back to unicode.
+    """
+    if not isinstance(s, text_type):
+        s = text_type(s)
+    return s

python/markupsafe/_speedups.c

@@ -0,0 +1,239 @@
+/**
+ * markupsafe._speedups
+ * ~~~~~~~~~~~~~~~~~~~~
+ *
+ * This module implements functions for automatic escaping in C for better
+ * performance.
+ *
+ * :copyright: (c) 2010 by Armin Ronacher.
+ * :license: BSD.
+ */
+
+#include <Python.h>
+
+#define ESCAPED_CHARS_TABLE_SIZE 63
+#define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL)));
+
+#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
+typedef int Py_ssize_t;
+#define PY_SSIZE_T_MAX INT_MAX
+#define PY_SSIZE_T_MIN INT_MIN
+#endif
+
+
+static PyObject* markup;
+static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE];
+static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE];
+
+static int
+init_constants(void)
+{
+	PyObject *module;
+	/* mapping of characters to replace */
+	escaped_chars_repl['"'] = UNICHR("&#34;");
+	escaped_chars_repl['\''] = UNICHR("&#39;");
+	escaped_chars_repl['&'] = UNICHR("&amp;");
+	escaped_chars_repl['<'] = UNICHR("&lt;");
+	escaped_chars_repl['>'] = UNICHR("&gt;");
+
+	/* lengths of those characters when replaced - 1 */
+	memset(escaped_chars_delta_len, 0, sizeof (escaped_chars_delta_len));
+	escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \
+		escaped_chars_delta_len['&'] = 4;
+	escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3;
+
+	/* import markup type so that we can mark the return value */
+	module = PyImport_ImportModule("markupsafe");
+	if (!module)
+		return 0;
+	markup = PyObject_GetAttrString(module, "Markup");
+	Py_DECREF(module);
+
+	return 1;
+}
+
+static PyObject*
+escape_unicode(PyUnicodeObject *in)
+{
+	PyUnicodeObject *out;
+	Py_UNICODE *inp = PyUnicode_AS_UNICODE(in);
+	const Py_UNICODE *inp_end = PyUnicode_AS_UNICODE(in) + PyUnicode_GET_SIZE(in);
+	Py_UNICODE *next_escp;
+	Py_UNICODE *outp;
+	Py_ssize_t delta=0, erepl=0, delta_len=0;
+
+	/* First we need to figure out how long the escaped string will be */
+	while (*(inp) || inp < inp_end) {
+		if (*inp < ESCAPED_CHARS_TABLE_SIZE) {
+			delta += escaped_chars_delta_len[*inp];
+			erepl += !!escaped_chars_delta_len[*inp];
+		}
+		++inp;
+	}
+
+	/* Do we need to escape anything at all? */
+	if (!erepl) {
+		Py_INCREF(in);
+		return (PyObject*)in;
+	}
+
+	out = (PyUnicodeObject*)PyUnicode_FromUnicode(NULL, PyUnicode_GET_SIZE(in) + delta);
+	if (!out)
+		return NULL;
+
+	outp = PyUnicode_AS_UNICODE(out);
+	inp = PyUnicode_AS_UNICODE(in);
+	while (erepl-- > 0) {
+		/* look for the next substitution */
+		next_escp = inp;
+		while (next_escp < inp_end) {
+			if (*next_escp < ESCAPED_CHARS_TABLE_SIZE &&
+			    (delta_len = escaped_chars_delta_len[*next_escp])) {
+				++delta_len;
+				break;
+			}
+			++next_escp;
+		}
+
+		if (next_escp > inp) {
+			/* copy unescaped chars between inp and next_escp */
+			Py_UNICODE_COPY(outp, inp, next_escp-inp);
+			outp += next_escp - inp;
+		}
+
+		/* escape 'next_escp' */
+		Py_UNICODE_COPY(outp, escaped_chars_repl[*next_escp], delta_len);
+		outp += delta_len;
+
+		inp = next_escp + 1;
+	}
+	if (inp < inp_end)
+		Py_UNICODE_COPY(outp, inp, PyUnicode_GET_SIZE(in) - (inp - PyUnicode_AS_UNICODE(in)));
+
+	return (PyObject*)out;
+}
+
+
+static PyObject*
+escape(PyObject *self, PyObject *text)
+{
+	PyObject *s = NULL, *rv = NULL, *html;
+
+	/* we don't have to escape integers, bools or floats */
+	if (PyLong_CheckExact(text) ||
+#if PY_MAJOR_VERSION < 3
+	    PyInt_CheckExact(text) ||
+#endif
+	    PyFloat_CheckExact(text) || PyBool_Check(text) ||
+	    text == Py_None)
+		return PyObject_CallFunctionObjArgs(markup, text, NULL);
+
+	/* if the object has an __html__ method that performs the escaping */
+	html = PyObject_GetAttrString(text, "__html__");
+	if (html) {
+		rv = PyObject_CallObject(html, NULL);
+		Py_DECREF(html);
+		return rv;
+	}
+
+	/* otherwise make the object unicode if it isn't, then escape */
+	PyErr_Clear();
+	if (!PyUnicode_Check(text)) {
+#if PY_MAJOR_VERSION < 3
+		PyObject *unicode = PyObject_Unicode(text);
+#else
+		PyObject *unicode = PyObject_Str(text);
+#endif
+		if (!unicode)
+			return NULL;
+		s = escape_unicode((PyUnicodeObject*)unicode);
+		Py_DECREF(unicode);
+	}
+	else
+		s = escape_unicode((PyUnicodeObject*)text);
+
+	/* convert the unicode string into a markup object. */
+	rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL);
+	Py_DECREF(s);
+	return rv;
+}
+
+
+static PyObject*
+escape_silent(PyObject *self, PyObject *text)
+{
+	if (text != Py_None)
+		return escape(self, text);
+	return PyObject_CallFunctionObjArgs(markup, NULL);
+}
+
+
+static PyObject*
+soft_unicode(PyObject *self, PyObject *s)
+{
+	if (!PyUnicode_Check(s))
+#if PY_MAJOR_VERSION < 3
+		return PyObject_Unicode(s);
+#else
+		return PyObject_Str(s);
+#endif
+	Py_INCREF(s);
+	return s;
+}
+
+
+static PyMethodDef module_methods[] = {
+	{"escape", (PyCFunction)escape, METH_O,
+	 "escape(s) -> markup\n\n"
+	 "Convert the characters &, <, >, ', and \" in string s to HTML-safe\n"
+	 "sequences.  Use this if you need to display text that might contain\n"
+	 "such characters in HTML.  Marks return value as markup string."},
+	{"escape_silent", (PyCFunction)escape_silent, METH_O,
+	 "escape_silent(s) -> markup\n\n"
+	 "Like escape but converts None to an empty string."},
+	{"soft_unicode", (PyCFunction)soft_unicode, METH_O,
+	 "soft_unicode(object) -> string\n\n"
+         "Make a string unicode if it isn't already.  That way a markup\n"
+         "string is not converted back to unicode."},
+	{NULL, NULL, 0, NULL}		/* Sentinel */
+};
+
+
+#if PY_MAJOR_VERSION < 3
+
+#ifndef PyMODINIT_FUNC	/* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+PyMODINIT_FUNC
+init_speedups(void)
+{
+	if (!init_constants())
+		return;
+
+	Py_InitModule3("markupsafe._speedups", module_methods, "");
+}
+
+#else /* Python 3.x module initialization */
+
+static struct PyModuleDef module_definition = {
+        PyModuleDef_HEAD_INIT,
+	"markupsafe._speedups",
+	NULL,
+	-1,
+	module_methods,
+	NULL,
+	NULL,
+	NULL,
+	NULL
+};
+
+PyMODINIT_FUNC
+PyInit__speedups(void)
+{
+	if (!init_constants())
+		return NULL;
+
+	return PyModule_Create(&module_definition);
+}
+
+#endif

python/werkzeug/__init__.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug
+    ~~~~~~~~
+
+    Werkzeug is the Swiss Army knife of Python web development.
+
+    It provides useful classes and functions for any WSGI application to make
+    the life of a python web developer much easier.  All of the provided
+    classes are independent from each other so you can mix it with any other
+    library.
+
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import sys
+from types import ModuleType
+
+__version__ = "0.15.4"
+
+# This import magic raises concerns quite often which is why the implementation
+# and motivation is explained here in detail now.
+#
+# The majority of the functions and classes provided by Werkzeug work on the
+# HTTP and WSGI layer.  There is no useful grouping for those which is why
+# they are all importable from "werkzeug" instead of the modules where they are
+# implemented.  The downside of that is, that now everything would be loaded at
+# once, even if unused.
+#
+# The implementation of a lazy-loading module in this file replaces the
+# werkzeug package when imported from within.  Attribute access to the werkzeug
+# module will then lazily import from the modules that implement the objects.
+
+# import mapping to objects in other modules
+all_by_module = {
+    "werkzeug.debug": ["DebuggedApplication"],
+    "werkzeug.local": [
+        "Local",
+        "LocalManager",
+        "LocalProxy",
+        "LocalStack",
+        "release_local",
+    ],
+    "werkzeug.serving": ["run_simple"],
+    "werkzeug.test": ["Client", "EnvironBuilder", "create_environ", "run_wsgi_app"],
+    "werkzeug.testapp": ["test_app"],
+    "werkzeug.exceptions": ["abort", "Aborter"],
+    "werkzeug.urls": [
+        "url_decode",
+        "url_encode",
+        "url_quote",
+        "url_quote_plus",
+        "url_unquote",
+        "url_unquote_plus",
+        "url_fix",
+        "Href",
+        "iri_to_uri",
+        "uri_to_iri",
+    ],
+    "werkzeug.formparser": ["parse_form_data"],
+    "werkzeug.utils": [
+        "escape",
+        "environ_property",
+        "append_slash_redirect",
+        "redirect",
+        "cached_property",
+        "import_string",
+        "dump_cookie",
+        "parse_cookie",
+        "unescape",
+        "format_string",
+        "find_modules",
+        "header_property",
+        "html",
+        "xhtml",
+        "HTMLBuilder",
+        "validate_arguments",
+        "ArgumentValidationError",
+        "bind_arguments",
+        "secure_filename",
+    ],
+    "werkzeug.wsgi": [
+        "get_current_url",
+        "get_host",
+        "pop_path_info",
+        "peek_path_info",
+        "ClosingIterator",
+        "FileWrapper",
+        "make_line_iter",
+        "LimitedStream",
+        "responder",
+        "wrap_file",
+        "extract_path_info",
+    ],
+    "werkzeug.datastructures": [
+        "MultiDict",
+        "CombinedMultiDict",
+        "Headers",
+        "EnvironHeaders",
+        "ImmutableList",
+        "ImmutableDict",
+        "ImmutableMultiDict",
+        "TypeConversionDict",
+        "ImmutableTypeConversionDict",
+        "Accept",
+        "MIMEAccept",
+        "CharsetAccept",
+        "LanguageAccept",
+        "RequestCacheControl",
+        "ResponseCacheControl",
+        "ETags",
+        "HeaderSet",
+        "WWWAuthenticate",
+        "Authorization",
+        "FileMultiDict",
+        "CallbackDict",
+        "FileStorage",
+        "OrderedMultiDict",
+        "ImmutableOrderedMultiDict",
+    ],
+    "werkzeug.useragents": ["UserAgent"],
+    "werkzeug.http": [
+        "parse_etags",
+        "parse_date",
+        "http_date",
+        "cookie_date",
+        "parse_cache_control_header",
+        "is_resource_modified",
+        "parse_accept_header",
+        "parse_set_header",
+        "quote_etag",
+        "unquote_etag",
+        "generate_etag",
+        "dump_header",
+        "parse_list_header",
+        "parse_dict_header",
+        "parse_authorization_header",
+        "parse_www_authenticate_header",
+        "remove_entity_headers",
+        "is_entity_header",
+        "remove_hop_by_hop_headers",
+        "parse_options_header",
+        "dump_options_header",
+        "is_hop_by_hop_header",
+        "unquote_header_value",
+        "quote_header_value",
+        "HTTP_STATUS_CODES",
+    ],
+    "werkzeug.wrappers": [
+        "BaseResponse",
+        "BaseRequest",
+        "Request",
+        "Response",
+        "AcceptMixin",
+        "ETagRequestMixin",
+        "ETagResponseMixin",
+        "ResponseStreamMixin",
+        "CommonResponseDescriptorsMixin",
+        "UserAgentMixin",
+        "AuthorizationMixin",
+        "WWWAuthenticateMixin",
+        "CommonRequestDescriptorsMixin",
+    ],
+    "werkzeug.middleware.dispatcher": ["DispatcherMiddleware"],
+    "werkzeug.middleware.shared_data": ["SharedDataMiddleware"],
+    "werkzeug.security": ["generate_password_hash", "check_password_hash"],
+    # the undocumented easteregg ;-)
+    "werkzeug._internal": ["_easteregg"],
+}
+
+# modules that should be imported when accessed as attributes of werkzeug
+attribute_modules = frozenset(["exceptions", "routing"])
+
+object_origins = {}
+for module, items in all_by_module.items():
+    for item in items:
+        object_origins[item] = module
+
+
+class module(ModuleType):
+    """Automatically import objects from the modules."""
+
+    def __getattr__(self, name):
+        if name in object_origins:
+            module = __import__(object_origins[name], None, None, [name])
+            for extra_name in all_by_module[module.__name__]:
+                setattr(self, extra_name, getattr(module, extra_name))
+            return getattr(module, name)
+        elif name in attribute_modules:
+            __import__("werkzeug." + name)
+        return ModuleType.__getattribute__(self, name)
+
+    def __dir__(self):
+        """Just show what we want to show."""
+        result = list(new_module.__all__)
+        result.extend(
+            (
+                "__file__",
+                "__doc__",
+                "__all__",
+                "__docformat__",
+                "__name__",
+                "__path__",
+                "__package__",
+                "__version__",
+            )
+        )
+        return result
+
+
+# keep a reference to this module so that it's not garbage collected
+old_module = sys.modules["werkzeug"]
+
+
+# setup the new module and patch it into the dict of loaded modules
+new_module = sys.modules["werkzeug"] = module("werkzeug")
+new_module.__dict__.update(
+    {
+        "__file__": __file__,
+        "__package__": "werkzeug",
+        "__path__": __path__,
+        "__doc__": __doc__,
+        "__version__": __version__,
+        "__all__": tuple(object_origins) + tuple(attribute_modules),
+        "__docformat__": "restructuredtext en",
+    }
+)
+
+
+# Due to bootstrapping issues we need to import exceptions here.
+# Don't ask :-(
+__import__("werkzeug.exceptions")

python/werkzeug/_compat.py

@@ -0,0 +1,219 @@
+# flake8: noqa
+# This whole file is full of lint errors
+import functools
+import operator
+import sys
+
+try:
+    import builtins
+except ImportError:
+    import __builtin__ as builtins
+
+
+PY2 = sys.version_info[0] == 2
+WIN = sys.platform.startswith("win")
+
+_identity = lambda x: x
+
+if PY2:
+    unichr = unichr
+    text_type = unicode
+    string_types = (str, unicode)
+    integer_types = (int, long)
+
+    iterkeys = lambda d, *args, **kwargs: d.iterkeys(*args, **kwargs)
+    itervalues = lambda d, *args, **kwargs: d.itervalues(*args, **kwargs)
+    iteritems = lambda d, *args, **kwargs: d.iteritems(*args, **kwargs)
+
+    iterlists = lambda d, *args, **kwargs: d.iterlists(*args, **kwargs)
+    iterlistvalues = lambda d, *args, **kwargs: d.iterlistvalues(*args, **kwargs)
+
+    int_to_byte = chr
+    iter_bytes = iter
+
+    import collections as collections_abc
+
+    exec("def reraise(tp, value, tb=None):\n raise tp, value, tb")
+
+    def fix_tuple_repr(obj):
+        def __repr__(self):
+            cls = self.__class__
+            return "%s(%s)" % (
+                cls.__name__,
+                ", ".join(
+                    "%s=%r" % (field, self[index])
+                    for index, field in enumerate(cls._fields)
+                ),
+            )
+
+        obj.__repr__ = __repr__
+        return obj
+
+    def implements_iterator(cls):
+        cls.next = cls.__next__
+        del cls.__next__
+        return cls
+
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode("utf-8")
+        return cls
+
+    def native_string_result(func):
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs).encode("utf-8")
+
+        return functools.update_wrapper(wrapper, func)
+
+    def implements_bool(cls):
+        cls.__nonzero__ = cls.__bool__
+        del cls.__bool__
+        return cls
+
+    from itertools import imap, izip, ifilter
+
+    range_type = xrange
+
+    from StringIO import StringIO
+    from cStringIO import StringIO as BytesIO
+
+    NativeStringIO = BytesIO
+
+    def make_literal_wrapper(reference):
+        return _identity
+
+    def normalize_string_tuple(tup):
+        """Normalizes a string tuple to a common type. Following Python 2
+        rules, upgrades to unicode are implicit.
+        """
+        if any(isinstance(x, text_type) for x in tup):
+            return tuple(to_unicode(x) for x in tup)
+        return tup
+
+    def try_coerce_native(s):
+        """Try to coerce a unicode string to native if possible. Otherwise,
+        leave it as unicode.
+        """
+        try:
+            return to_native(s)
+        except UnicodeError:
+            return s
+
+    wsgi_get_bytes = _identity
+
+    def wsgi_decoding_dance(s, charset="utf-8", errors="replace"):
+        return s.decode(charset, errors)
+
+    def wsgi_encoding_dance(s, charset="utf-8", errors="replace"):
+        if isinstance(s, bytes):
+            return s
+        return s.encode(charset, errors)
+
+    def to_bytes(x, charset=sys.getdefaultencoding(), errors="strict"):
+        if x is None:
+            return None
+        if isinstance(x, (bytes, bytearray, buffer)):
+            return bytes(x)
+        if isinstance(x, unicode):
+            return x.encode(charset, errors)
+        raise TypeError("Expected bytes")
+
+    def to_native(x, charset=sys.getdefaultencoding(), errors="strict"):
+        if x is None or isinstance(x, str):
+            return x
+        return x.encode(charset, errors)
+
+
+else:
+    unichr = chr
+    text_type = str
+    string_types = (str,)
+    integer_types = (int,)
+
+    iterkeys = lambda d, *args, **kwargs: iter(d.keys(*args, **kwargs))
+    itervalues = lambda d, *args, **kwargs: iter(d.values(*args, **kwargs))
+    iteritems = lambda d, *args, **kwargs: iter(d.items(*args, **kwargs))
+
+    iterlists = lambda d, *args, **kwargs: iter(d.lists(*args, **kwargs))
+    iterlistvalues = lambda d, *args, **kwargs: iter(d.listvalues(*args, **kwargs))
+
+    int_to_byte = operator.methodcaller("to_bytes", 1, "big")
+    iter_bytes = functools.partial(map, int_to_byte)
+
+    import collections.abc as collections_abc
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+    fix_tuple_repr = _identity
+    implements_iterator = _identity
+    implements_to_string = _identity
+    implements_bool = _identity
+    native_string_result = _identity
+    imap = map
+    izip = zip
+    ifilter = filter
+    range_type = range
+
+    from io import StringIO, BytesIO
+
+    NativeStringIO = StringIO
+
+    _latin1_encode = operator.methodcaller("encode", "latin1")
+
+    def make_literal_wrapper(reference):
+        if isinstance(reference, text_type):
+            return _identity
+        return _latin1_encode
+
+    def normalize_string_tuple(tup):
+        """Ensures that all types in the tuple are either strings
+        or bytes.
+        """
+        tupiter = iter(tup)
+        is_text = isinstance(next(tupiter, None), text_type)
+        for arg in tupiter:
+            if isinstance(arg, text_type) != is_text:
+                raise TypeError(
+                    "Cannot mix str and bytes arguments (got %s)" % repr(tup)
+                )
+        return tup
+
+    try_coerce_native = _identity
+    wsgi_get_bytes = _latin1_encode
+
+    def wsgi_decoding_dance(s, charset="utf-8", errors="replace"):
+        return s.encode("latin1").decode(charset, errors)
+
+    def wsgi_encoding_dance(s, charset="utf-8", errors="replace"):
+        if isinstance(s, text_type):
+            s = s.encode(charset)
+        return s.decode("latin1", errors)
+
+    def to_bytes(x, charset=sys.getdefaultencoding(), errors="strict"):
+        if x is None:
+            return None
+        if isinstance(x, (bytes, bytearray, memoryview)):  # noqa
+            return bytes(x)
+        if isinstance(x, str):
+            return x.encode(charset, errors)
+        raise TypeError("Expected bytes")
+
+    def to_native(x, charset=sys.getdefaultencoding(), errors="strict"):
+        if x is None or isinstance(x, str):
+            return x
+        return x.decode(charset, errors)
+
+
+def to_unicode(
+    x, charset=sys.getdefaultencoding(), errors="strict", allow_none_charset=False
+):
+    if x is None:
+        return None
+    if not isinstance(x, bytes):
+        return text_type(x)
+    if charset is None and allow_none_charset:
+        return x
+    return x.decode(charset, errors)

python/werkzeug/_internal.py

@@ -0,0 +1,484 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug._internal
+    ~~~~~~~~~~~~~~~~~~
+
+    This module provides internally used helpers and constants.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import inspect
+import logging
+import re
+import string
+from datetime import date
+from datetime import datetime
+from itertools import chain
+from weakref import WeakKeyDictionary
+
+from ._compat import int_to_byte
+from ._compat import integer_types
+from ._compat import iter_bytes
+from ._compat import range_type
+from ._compat import text_type
+
+
+_logger = None
+_signature_cache = WeakKeyDictionary()
+_epoch_ord = date(1970, 1, 1).toordinal()
+_cookie_params = {
+    b"expires",
+    b"path",
+    b"comment",
+    b"max-age",
+    b"secure",
+    b"httponly",
+    b"version",
+}
+_legal_cookie_chars = (
+    string.ascii_letters + string.digits + u"/=!#$%&'*+-.^_`|~:"
+).encode("ascii")
+
+_cookie_quoting_map = {b",": b"\\054", b";": b"\\073", b'"': b'\\"', b"\\": b"\\\\"}
+for _i in chain(range_type(32), range_type(127, 256)):
+    _cookie_quoting_map[int_to_byte(_i)] = ("\\%03o" % _i).encode("latin1")
+
+_octal_re = re.compile(br"\\[0-3][0-7][0-7]")
+_quote_re = re.compile(br"[\\].")
+_legal_cookie_chars_re = br"[\w\d!#%&\'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_cookie_re = re.compile(
+    br"""
+    (?P<key>[^=;]+)
+    (?:\s*=\s*
+        (?P<val>
+            "(?:[^\\"]|\\.)*" |
+             (?:.*?)
+        )
+    )?
+    \s*;
+""",
+    flags=re.VERBOSE,
+)
+
+
+class _Missing(object):
+    def __repr__(self):
+        return "no value"
+
+    def __reduce__(self):
+        return "_missing"
+
+
+_missing = _Missing()
+
+
+def _get_environ(obj):
+    env = getattr(obj, "environ", obj)
+    assert isinstance(env, dict), (
+        "%r is not a WSGI environment (has to be a dict)" % type(obj).__name__
+    )
+    return env
+
+
+def _has_level_handler(logger):
+    """Check if there is a handler in the logging chain that will handle
+    the given logger's effective level.
+    """
+    level = logger.getEffectiveLevel()
+    current = logger
+
+    while current:
+        if any(handler.level <= level for handler in current.handlers):
+            return True
+
+        if not current.propagate:
+            break
+
+        current = current.parent
+
+    return False
+
+
+def _log(type, message, *args, **kwargs):
+    """Log a message to the 'werkzeug' logger.
+
+    The logger is created the first time it is needed. If there is no
+    level set, it is set to :data:`logging.INFO`. If there is no handler
+    for the logger's effective level, a :class:`logging.StreamHandler`
+    is added.
+    """
+    global _logger
+
+    if _logger is None:
+        _logger = logging.getLogger("werkzeug")
+
+        if _logger.level == logging.NOTSET:
+            _logger.setLevel(logging.INFO)
+
+        if not _has_level_handler(_logger):
+            _logger.addHandler(logging.StreamHandler())
+
+    getattr(_logger, type)(message.rstrip(), *args, **kwargs)
+
+
+def _parse_signature(func):
+    """Return a signature object for the function."""
+    if hasattr(func, "im_func"):
+        func = func.im_func
+
+    # if we have a cached validator for this function, return it
+    parse = _signature_cache.get(func)
+    if parse is not None:
+        return parse
+
+    # inspect the function signature and collect all the information
+    if hasattr(inspect, "getfullargspec"):
+        tup = inspect.getfullargspec(func)
+    else:
+        tup = inspect.getargspec(func)
+    positional, vararg_var, kwarg_var, defaults = tup[:4]
+    defaults = defaults or ()
+    arg_count = len(positional)
+    arguments = []
+    for idx, name in enumerate(positional):
+        if isinstance(name, list):
+            raise TypeError(
+                "cannot parse functions that unpack tuples in the function signature"
+            )
+        try:
+            default = defaults[idx - arg_count]
+        except IndexError:
+            param = (name, False, None)
+        else:
+            param = (name, True, default)
+        arguments.append(param)
+    arguments = tuple(arguments)
+
+    def parse(args, kwargs):
+        new_args = []
+        missing = []
+        extra = {}
+
+        # consume as many arguments as positional as possible
+        for idx, (name, has_default, default) in enumerate(arguments):
+            try:
+                new_args.append(args[idx])
+            except IndexError:
+                try:
+                    new_args.append(kwargs.pop(name))
+                except KeyError:
+                    if has_default:
+                        new_args.append(default)
+                    else:
+                        missing.append(name)
+            else:
+                if name in kwargs:
+                    extra[name] = kwargs.pop(name)
+
+        # handle extra arguments
+        extra_positional = args[arg_count:]
+        if vararg_var is not None:
+            new_args.extend(extra_positional)
+            extra_positional = ()
+        if kwargs and kwarg_var is None:
+            extra.update(kwargs)
+            kwargs = {}
+
+        return (
+            new_args,
+            kwargs,
+            missing,
+            extra,
+            extra_positional,
+            arguments,
+            vararg_var,
+            kwarg_var,
+        )
+
+    _signature_cache[func] = parse
+    return parse
+
+
+def _date_to_unix(arg):
+    """Converts a timetuple, integer or datetime object into the seconds from
+    epoch in utc.
+    """
+    if isinstance(arg, datetime):
+        arg = arg.utctimetuple()
+    elif isinstance(arg, integer_types + (float,)):
+        return int(arg)
+    year, month, day, hour, minute, second = arg[:6]
+    days = date(year, month, 1).toordinal() - _epoch_ord + day - 1
+    hours = days * 24 + hour
+    minutes = hours * 60 + minute
+    seconds = minutes * 60 + second
+    return seconds
+
+
+class _DictAccessorProperty(object):
+    """Baseclass for `environ_property` and `header_property`."""
+
+    read_only = False
+
+    def __init__(
+        self,
+        name,
+        default=None,
+        load_func=None,
+        dump_func=None,
+        read_only=None,
+        doc=None,
+    ):
+        self.name = name
+        self.default = default
+        self.load_func = load_func
+        self.dump_func = dump_func
+        if read_only is not None:
+            self.read_only = read_only
+        self.__doc__ = doc
+
+    def __get__(self, obj, type=None):
+        if obj is None:
+            return self
+        storage = self.lookup(obj)
+        if self.name not in storage:
+            return self.default
+        rv = storage[self.name]
+        if self.load_func is not None:
+            try:
+                rv = self.load_func(rv)
+            except (ValueError, TypeError):
+                rv = self.default
+        return rv
+
+    def __set__(self, obj, value):
+        if self.read_only:
+            raise AttributeError("read only property")
+        if self.dump_func is not None:
+            value = self.dump_func(value)
+        self.lookup(obj)[self.name] = value
+
+    def __delete__(self, obj):
+        if self.read_only:
+            raise AttributeError("read only property")
+        self.lookup(obj).pop(self.name, None)
+
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, self.name)
+
+
+def _cookie_quote(b):
+    buf = bytearray()
+    all_legal = True
+    _lookup = _cookie_quoting_map.get
+    _push = buf.extend
+
+    for char in iter_bytes(b):
+        if char not in _legal_cookie_chars:
+            all_legal = False
+            char = _lookup(char, char)
+        _push(char)
+
+    if all_legal:
+        return bytes(buf)
+    return bytes(b'"' + buf + b'"')
+
+
+def _cookie_unquote(b):
+    if len(b) < 2:
+        return b
+    if b[:1] != b'"' or b[-1:] != b'"':
+        return b
+
+    b = b[1:-1]
+
+    i = 0
+    n = len(b)
+    rv = bytearray()
+    _push = rv.extend
+
+    while 0 <= i < n:
+        o_match = _octal_re.search(b, i)
+        q_match = _quote_re.search(b, i)
+        if not o_match and not q_match:
+            rv.extend(b[i:])
+            break
+        j = k = -1
+        if o_match:
+            j = o_match.start(0)
+        if q_match:
+            k = q_match.start(0)
+        if q_match and (not o_match or k < j):
+            _push(b[i:k])
+            _push(b[k + 1 : k + 2])
+            i = k + 2
+        else:
+            _push(b[i:j])
+            rv.append(int(b[j + 1 : j + 4], 8))
+            i = j + 4
+
+    return bytes(rv)
+
+
+def _cookie_parse_impl(b):
+    """Lowlevel cookie parsing facility that operates on bytes."""
+    i = 0
+    n = len(b)
+
+    while i < n:
+        match = _cookie_re.search(b + b";", i)
+        if not match:
+            break
+
+        key = match.group("key").strip()
+        value = match.group("val") or b""
+        i = match.end(0)
+
+        # Ignore parameters.  We have no interest in them.
+        if key.lower() not in _cookie_params:
+            yield _cookie_unquote(key), _cookie_unquote(value)
+
+
+def _encode_idna(domain):
+    # If we're given bytes, make sure they fit into ASCII
+    if not isinstance(domain, text_type):
+        domain.decode("ascii")
+        return domain
+
+    # Otherwise check if it's already ascii, then return
+    try:
+        return domain.encode("ascii")
+    except UnicodeError:
+        pass
+
+    # Otherwise encode each part separately
+    parts = domain.split(".")
+    for idx, part in enumerate(parts):
+        parts[idx] = part.encode("idna")
+    return b".".join(parts)
+
+
+def _decode_idna(domain):
+    # If the input is a string try to encode it to ascii to
+    # do the idna decoding.  if that fails because of an
+    # unicode error, then we already have a decoded idna domain
+    if isinstance(domain, text_type):
+        try:
+            domain = domain.encode("ascii")
+        except UnicodeError:
+            return domain
+
+    # Decode each part separately.  If a part fails, try to
+    # decode it with ascii and silently ignore errors.  This makes
+    # most sense because the idna codec does not have error handling
+    parts = domain.split(b".")
+    for idx, part in enumerate(parts):
+        try:
+            parts[idx] = part.decode("idna")
+        except UnicodeError:
+            parts[idx] = part.decode("ascii", "ignore")
+
+    return ".".join(parts)
+
+
+def _make_cookie_domain(domain):
+    if domain is None:
+        return None
+    domain = _encode_idna(domain)
+    if b":" in domain:
+        domain = domain.split(b":", 1)[0]
+    if b"." in domain:
+        return domain
+    raise ValueError(
+        "Setting 'domain' for a cookie on a server running locally (ex: "
+        "localhost) is not supported by complying browsers. You should "
+        "have something like: '127.0.0.1 localhost dev.localhost' on "
+        "your hosts file and then point your server to run on "
+        "'dev.localhost' and also set 'domain' for 'dev.localhost'"
+    )
+
+
+def _easteregg(app=None):
+    """Like the name says.  But who knows how it works?"""
+
+    def bzzzzzzz(gyver):
+        import base64
+        import zlib
+
+        return zlib.decompress(base64.b64decode(gyver)).decode("ascii")
+
+    gyver = u"\n".join(
+        [
+            x + (77 - len(x)) * u" "
+            for x in bzzzzzzz(
+                b"""
+eJyFlzuOJDkMRP06xRjymKgDJCDQStBYT8BCgK4gTwfQ2fcFs2a2FzvZk+hvlcRvRJD148efHt9m
+9Xz94dRY5hGt1nrYcXx7us9qlcP9HHNh28rz8dZj+q4rynVFFPdlY4zH873NKCexrDM6zxxRymzz
+4QIxzK4bth1PV7+uHn6WXZ5C4ka/+prFzx3zWLMHAVZb8RRUxtFXI5DTQ2n3Hi2sNI+HK43AOWSY
+jmEzE4naFp58PdzhPMdslLVWHTGUVpSxImw+pS/D+JhzLfdS1j7PzUMxij+mc2U0I9zcbZ/HcZxc
+q1QjvvcThMYFnp93agEx392ZdLJWXbi/Ca4Oivl4h/Y1ErEqP+lrg7Xa4qnUKu5UE9UUA4xeqLJ5
+jWlPKJvR2yhRI7xFPdzPuc6adXu6ovwXwRPXXnZHxlPtkSkqWHilsOrGrvcVWXgGP3daXomCj317
+8P2UOw/NnA0OOikZyFf3zZ76eN9QXNwYdD8f8/LdBRFg0BO3bB+Pe/+G8er8tDJv83XTkj7WeMBJ
+v/rnAfdO51d6sFglfi8U7zbnr0u9tyJHhFZNXYfH8Iafv2Oa+DT6l8u9UYlajV/hcEgk1x8E8L/r
+XJXl2SK+GJCxtnyhVKv6GFCEB1OO3f9YWAIEbwcRWv/6RPpsEzOkXURMN37J0PoCSYeBnJQd9Giu
+LxYQJNlYPSo/iTQwgaihbART7Fcyem2tTSCcwNCs85MOOpJtXhXDe0E7zgZJkcxWTar/zEjdIVCk
+iXy87FW6j5aGZhttDBoAZ3vnmlkx4q4mMmCdLtnHkBXFMCReqthSGkQ+MDXLLCpXwBs0t+sIhsDI
+tjBB8MwqYQpLygZ56rRHHpw+OAVyGgaGRHWy2QfXez+ZQQTTBkmRXdV/A9LwH6XGZpEAZU8rs4pE
+1R4FQ3Uwt8RKEtRc0/CrANUoes3EzM6WYcFyskGZ6UTHJWenBDS7h163Eo2bpzqxNE9aVgEM2CqI
+GAJe9Yra4P5qKmta27VjzYdR04Vc7KHeY4vs61C0nbywFmcSXYjzBHdiEjraS7PGG2jHHTpJUMxN
+Jlxr3pUuFvlBWLJGE3GcA1/1xxLcHmlO+LAXbhrXah1tD6Ze+uqFGdZa5FM+3eHcKNaEarutAQ0A
+QMAZHV+ve6LxAwWnXbbSXEG2DmCX5ijeLCKj5lhVFBrMm+ryOttCAeFpUdZyQLAQkA06RLs56rzG
+8MID55vqr/g64Qr/wqwlE0TVxgoiZhHrbY2h1iuuyUVg1nlkpDrQ7Vm1xIkI5XRKLedN9EjzVchu
+jQhXcVkjVdgP2O99QShpdvXWoSwkp5uMwyjt3jiWCqWGSiaaPAzohjPanXVLbM3x0dNskJsaCEyz
+DTKIs+7WKJD4ZcJGfMhLFBf6hlbnNkLEePF8Cx2o2kwmYF4+MzAxa6i+6xIQkswOqGO+3x9NaZX8
+MrZRaFZpLeVTYI9F/djY6DDVVs340nZGmwrDqTCiiqD5luj3OzwpmQCiQhdRYowUYEA3i1WWGwL4
+GCtSoO4XbIPFeKGU13XPkDf5IdimLpAvi2kVDVQbzOOa4KAXMFlpi/hV8F6IDe0Y2reg3PuNKT3i
+RYhZqtkQZqSB2Qm0SGtjAw7RDwaM1roESC8HWiPxkoOy0lLTRFG39kvbLZbU9gFKFRvixDZBJmpi
+Xyq3RE5lW00EJjaqwp/v3EByMSpVZYsEIJ4APaHmVtpGSieV5CALOtNUAzTBiw81GLgC0quyzf6c
+NlWknzJeCsJ5fup2R4d8CYGN77mu5vnO1UqbfElZ9E6cR6zbHjgsr9ly18fXjZoPeDjPuzlWbFwS
+pdvPkhntFvkc13qb9094LL5NrA3NIq3r9eNnop9DizWOqCEbyRBFJTHn6Tt3CG1o8a4HevYh0XiJ
+sR0AVVHuGuMOIfbuQ/OKBkGRC6NJ4u7sbPX8bG/n5sNIOQ6/Y/BX3IwRlTSabtZpYLB85lYtkkgm
+p1qXK3Du2mnr5INXmT/78KI12n11EFBkJHHp0wJyLe9MvPNUGYsf+170maayRoy2lURGHAIapSpQ
+krEDuNoJCHNlZYhKpvw4mspVWxqo415n8cD62N9+EfHrAvqQnINStetek7RY2Urv8nxsnGaZfRr/
+nhXbJ6m/yl1LzYqscDZA9QHLNbdaSTTr+kFg3bC0iYbX/eQy0Bv3h4B50/SGYzKAXkCeOLI3bcAt
+mj2Z/FM1vQWgDynsRwNvrWnJHlespkrp8+vO1jNaibm+PhqXPPv30YwDZ6jApe3wUjFQobghvW9p
+7f2zLkGNv8b191cD/3vs9Q833z8t"""
+            ).splitlines()
+        ]
+    )
+
+    def easteregged(environ, start_response):
+        def injecting_start_response(status, headers, exc_info=None):
+            headers.append(("X-Powered-By", "Werkzeug"))
+            return start_response(status, headers, exc_info)
+
+        if app is not None and environ.get("QUERY_STRING") != "macgybarchakku":
+            return app(environ, injecting_start_response)
+        injecting_start_response("200 OK", [("Content-Type", "text/html")])
+        return [
+            (
+                u"""
+<!DOCTYPE html>
+<html>
+<head>
+<title>About Werkzeug</title>
+<style type="text/css">
+  body { font: 15px Georgia, serif; text-align: center; }
+  a { color: #333; text-decoration: none; }
+  h1 { font-size: 30px; margin: 20px 0 10px 0; }
+  p { margin: 0 0 30px 0; }
+  pre { font: 11px 'Consolas', 'Monaco', monospace; line-height: 0.95; }
+</style>
+</head>
+<body>
+<h1><a href="http://werkzeug.pocoo.org/">Werkzeug</a></h1>
+<p>the Swiss Army knife of Python web development.</p>
+<pre>%s\n\n\n</pre>
+</body>
+</html>"""
+                % gyver
+            ).encode("latin1")
+        ]
+
+    return easteregged

python/werkzeug/_reloader.py

@@ -0,0 +1,334 @@
+import os
+import subprocess
+import sys
+import threading
+import time
+from itertools import chain
+
+from ._compat import iteritems
+from ._compat import PY2
+from ._compat import text_type
+from ._internal import _log
+
+
+def _iter_module_files():
+    """This iterates over all relevant Python files.  It goes through all
+    loaded files from modules, all files in folders of already loaded modules
+    as well as all files reachable through a package.
+    """
+    # The list call is necessary on Python 3 in case the module
+    # dictionary modifies during iteration.
+    for module in list(sys.modules.values()):
+        if module is None:
+            continue
+        filename = getattr(module, "__file__", None)
+        if filename:
+            if os.path.isdir(filename) and os.path.exists(
+                os.path.join(filename, "__init__.py")
+            ):
+                filename = os.path.join(filename, "__init__.py")
+
+            old = None
+            while not os.path.isfile(filename):
+                old = filename
+                filename = os.path.dirname(filename)
+                if filename == old:
+                    break
+            else:
+                if filename[-4:] in (".pyc", ".pyo"):
+                    filename = filename[:-1]
+                yield filename
+
+
+def _find_observable_paths(extra_files=None):
+    """Finds all paths that should be observed."""
+    rv = set(
+        os.path.dirname(os.path.abspath(x)) if os.path.isfile(x) else os.path.abspath(x)
+        for x in sys.path
+    )
+
+    for filename in extra_files or ():
+        rv.add(os.path.dirname(os.path.abspath(filename)))
+
+    for module in list(sys.modules.values()):
+        fn = getattr(module, "__file__", None)
+        if fn is None:
+            continue
+        fn = os.path.abspath(fn)
+        rv.add(os.path.dirname(fn))
+
+    return _find_common_roots(rv)
+
+
+def _get_args_for_reloading():
+    """Returns the executable. This contains a workaround for windows
+    if the executable is incorrectly reported to not have the .exe
+    extension which can cause bugs on reloading.  This also contains
+    a workaround for linux where the file is executable (possibly with
+    a program other than python)
+    """
+    rv = [sys.executable]
+    py_script = os.path.abspath(sys.argv[0])
+    args = sys.argv[1:]
+    # Need to look at main module to determine how it was executed.
+    __main__ = sys.modules["__main__"]
+
+    if __main__.__package__ is None:
+        # Executed a file, like "python app.py".
+        if os.name == "nt":
+            # Windows entry points have ".exe" extension and should be
+            # called directly.
+            if not os.path.exists(py_script) and os.path.exists(py_script + ".exe"):
+                py_script += ".exe"
+
+            if (
+                os.path.splitext(rv[0])[1] == ".exe"
+                and os.path.splitext(py_script)[1] == ".exe"
+            ):
+                rv.pop(0)
+
+        elif os.path.isfile(py_script) and os.access(py_script, os.X_OK):
+            # The file is marked as executable. Nix adds a wrapper that
+            # shouldn't be called with the Python executable.
+            rv.pop(0)
+
+        rv.append(py_script)
+    else:
+        # Executed a module, like "python -m werkzeug.serving".
+        if sys.argv[0] == "-m":
+            # Flask works around previous behavior by putting
+            # "-m flask" in sys.argv.
+            # TODO remove this once Flask no longer misbehaves
+            args = sys.argv
+        else:
+            py_module = __main__.__package__
+            name = os.path.splitext(os.path.basename(py_script))[0]
+
+            if name != "__main__":
+                py_module += "." + name
+
+            rv.extend(("-m", py_module.lstrip(".")))
+
+    rv.extend(args)
+    return rv
+
+
+def _find_common_roots(paths):
+    """Out of some paths it finds the common roots that need monitoring."""
+    paths = [x.split(os.path.sep) for x in paths]
+    root = {}
+    for chunks in sorted(paths, key=len, reverse=True):
+        node = root
+        for chunk in chunks:
+            node = node.setdefault(chunk, {})
+        node.clear()
+
+    rv = set()
+
+    def _walk(node, path):
+        for prefix, child in iteritems(node):
+            _walk(child, path + (prefix,))
+        if not node:
+            rv.add("/".join(path))
+
+    _walk(root, ())
+    return rv
+
+
+class ReloaderLoop(object):
+    name = None
+
+    # monkeypatched by testsuite. wrapping with `staticmethod` is required in
+    # case time.sleep has been replaced by a non-c function (e.g. by
+    # `eventlet.monkey_patch`) before we get here
+    _sleep = staticmethod(time.sleep)
+
+    def __init__(self, extra_files=None, interval=1):
+        self.extra_files = set(os.path.abspath(x) for x in extra_files or ())
+        self.interval = interval
+
+    def run(self):
+        pass
+
+    def restart_with_reloader(self):
+        """Spawn a new Python interpreter with the same arguments as this one,
+        but running the reloader thread.
+        """
+        while 1:
+            _log("info", " * Restarting with %s" % self.name)
+            args = _get_args_for_reloading()
+
+            # a weird bug on windows. sometimes unicode strings end up in the
+            # environment and subprocess.call does not like this, encode them
+            # to latin1 and continue.
+            if os.name == "nt" and PY2:
+                new_environ = {}
+                for key, value in iteritems(os.environ):
+                    if isinstance(key, text_type):
+                        key = key.encode("iso-8859-1")
+                    if isinstance(value, text_type):
+                        value = value.encode("iso-8859-1")
+                    new_environ[key] = value
+            else:
+                new_environ = os.environ.copy()
+
+            new_environ["WERKZEUG_RUN_MAIN"] = "true"
+            exit_code = subprocess.call(args, env=new_environ, close_fds=False)
+            if exit_code != 3:
+                return exit_code
+
+    def trigger_reload(self, filename):
+        self.log_reload(filename)
+        sys.exit(3)
+
+    def log_reload(self, filename):
+        filename = os.path.abspath(filename)
+        _log("info", " * Detected change in %r, reloading" % filename)
+
+
+class StatReloaderLoop(ReloaderLoop):
+    name = "stat"
+
+    def run(self):
+        mtimes = {}
+        while 1:
+            for filename in chain(_iter_module_files(), self.extra_files):
+                try:
+                    mtime = os.stat(filename).st_mtime
+                except OSError:
+                    continue
+
+                old_time = mtimes.get(filename)
+                if old_time is None:
+                    mtimes[filename] = mtime
+                    continue
+                elif mtime > old_time:
+                    self.trigger_reload(filename)
+            self._sleep(self.interval)
+
+
+class WatchdogReloaderLoop(ReloaderLoop):
+    def __init__(self, *args, **kwargs):
+        ReloaderLoop.__init__(self, *args, **kwargs)
+        from watchdog.observers import Observer
+        from watchdog.events import FileSystemEventHandler
+
+        self.observable_paths = set()
+
+        def _check_modification(filename):
+            if filename in self.extra_files:
+                self.trigger_reload(filename)
+            dirname = os.path.dirname(filename)
+            if dirname.startswith(tuple(self.observable_paths)):
+                if filename.endswith((".pyc", ".pyo", ".py")):
+                    self.trigger_reload(filename)
+
+        class _CustomHandler(FileSystemEventHandler):
+            def on_created(self, event):
+                _check_modification(event.src_path)
+
+            def on_modified(self, event):
+                _check_modification(event.src_path)
+
+            def on_moved(self, event):
+                _check_modification(event.src_path)
+                _check_modification(event.dest_path)
+
+            def on_deleted(self, event):
+                _check_modification(event.src_path)
+
+        reloader_name = Observer.__name__.lower()
+        if reloader_name.endswith("observer"):
+            reloader_name = reloader_name[:-8]
+        reloader_name += " reloader"
+
+        self.name = reloader_name
+
+        self.observer_class = Observer
+        self.event_handler = _CustomHandler()
+        self.should_reload = False
+
+    def trigger_reload(self, filename):
+        # This is called inside an event handler, which means throwing
+        # SystemExit has no effect.
+        # https://github.com/gorakhargosh/watchdog/issues/294
+        self.should_reload = True
+        self.log_reload(filename)
+
+    def run(self):
+        watches = {}
+        observer = self.observer_class()
+        observer.start()
+
+        try:
+            while not self.should_reload:
+                to_delete = set(watches)
+                paths = _find_observable_paths(self.extra_files)
+                for path in paths:
+                    if path not in watches:
+                        try:
+                            watches[path] = observer.schedule(
+                                self.event_handler, path, recursive=True
+                            )
+                        except OSError:
+                            # Clear this path from list of watches We don't want
+                            # the same error message showing again in the next
+                            # iteration.
+                            watches[path] = None
+                    to_delete.discard(path)
+                for path in to_delete:
+                    watch = watches.pop(path, None)
+                    if watch is not None:
+                        observer.unschedule(watch)
+                self.observable_paths = paths
+                self._sleep(self.interval)
+        finally:
+            observer.stop()
+            observer.join()
+
+        sys.exit(3)
+
+
+reloader_loops = {"stat": StatReloaderLoop, "watchdog": WatchdogReloaderLoop}
+
+try:
+    __import__("watchdog.observers")
+except ImportError:
+    reloader_loops["auto"] = reloader_loops["stat"]
+else:
+    reloader_loops["auto"] = reloader_loops["watchdog"]
+
+
+def ensure_echo_on():
+    """Ensure that echo mode is enabled. Some tools such as PDB disable
+    it which causes usability issues after reload."""
+    # tcgetattr will fail if stdin isn't a tty
+    if not sys.stdin.isatty():
+        return
+    try:
+        import termios
+    except ImportError:
+        return
+    attributes = termios.tcgetattr(sys.stdin)
+    if not attributes[3] & termios.ECHO:
+        attributes[3] |= termios.ECHO
+        termios.tcsetattr(sys.stdin, termios.TCSANOW, attributes)
+
+
+def run_with_reloader(main_func, extra_files=None, interval=1, reloader_type="auto"):
+    """Run the given function in an independent python interpreter."""
+    import signal
+
+    reloader = reloader_loops[reloader_type](extra_files, interval)
+    signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
+    try:
+        if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+            ensure_echo_on()
+            t = threading.Thread(target=main_func, args=())
+            t.setDaemon(True)
+            t.start()
+            reloader.run()
+        else:
+            sys.exit(reloader.restart_with_reloader())
+    except KeyboardInterrupt:
+        pass

python/werkzeug/contrib/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.contrib
+    ~~~~~~~~~~~~~~~~
+
+    Contains user-submitted code that other users may find useful, but which
+    is not part of the Werkzeug core.  Anyone can write code for inclusion in
+    the `contrib` package.  All modules in this package are distributed as an
+    add-on library and thus are not part of Werkzeug itself.
+
+    This file itself is mostly for informational purposes and to tell the
+    Python interpreter that `contrib` is a package.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""

python/werkzeug/contrib/atom.py

@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.contrib.atom
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    This module provides a class called :class:`AtomFeed` which can be
+    used to generate feeds in the Atom syndication format (see :rfc:`4287`).
+
+    Example::
+
+        def atom_feed(request):
+            feed = AtomFeed("My Blog", feed_url=request.url,
+                            url=request.host_url,
+                            subtitle="My example blog for a feed test.")
+            for post in Post.query.limit(10).all():
+                feed.add(post.title, post.body, content_type='html',
+                         author=post.author, url=post.url, id=post.uid,
+                         updated=post.last_update, published=post.pub_date)
+            return feed.get_response()
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import warnings
+from datetime import datetime
+
+from .._compat import implements_to_string
+from .._compat import string_types
+from ..utils import escape
+from ..wrappers import BaseResponse
+
+warnings.warn(
+    "'werkzeug.contrib.atom' is deprecated as of version 0.15 and will"
+    " be removed in version 1.0.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+
+
+def _make_text_block(name, content, content_type=None):
+    """Helper function for the builder that creates an XML text block."""
+    if content_type == "xhtml":
+        return u'<%s type="xhtml"><div xmlns="%s">%s</div></%s>\n' % (
+            name,
+            XHTML_NAMESPACE,
+            content,
+            name,
+        )
+    if not content_type:
+        return u"<%s>%s</%s>\n" % (name, escape(content), name)
+    return u'<%s type="%s">%s</%s>\n' % (name, content_type, escape(content), name)
+
+
+def format_iso8601(obj):
+    """Format a datetime object for iso8601"""
+    iso8601 = obj.isoformat()
+    if obj.tzinfo:
+        return iso8601
+    return iso8601 + "Z"
+
+
+@implements_to_string
+class AtomFeed(object):
+
+    """A helper class that creates Atom feeds.
+
+    :param title: the title of the feed. Required.
+    :param title_type: the type attribute for the title element.  One of
+                       ``'html'``, ``'text'`` or ``'xhtml'``.
+    :param url: the url for the feed (not the url *of* the feed)
+    :param id: a globally unique id for the feed.  Must be an URI.  If
+               not present the `feed_url` is used, but one of both is
+               required.
+    :param updated: the time the feed was modified the last time.  Must
+                    be a :class:`datetime.datetime` object.  If not
+                    present the latest entry's `updated` is used.
+                    Treated as UTC if naive datetime.
+    :param feed_url: the URL to the feed.  Should be the URL that was
+                     requested.
+    :param author: the author of the feed.  Must be either a string (the
+                   name) or a dict with name (required) and uri or
+                   email (both optional).  Can be a list of (may be
+                   mixed, too) strings and dicts, too, if there are
+                   multiple authors. Required if not every entry has an
+                   author element.
+    :param icon: an icon for the feed.
+    :param logo: a logo for the feed.
+    :param rights: copyright information for the feed.
+    :param rights_type: the type attribute for the rights element.  One of
+                        ``'html'``, ``'text'`` or ``'xhtml'``.  Default is
+                        ``'text'``.
+    :param subtitle: a short description of the feed.
+    :param subtitle_type: the type attribute for the subtitle element.
+                          One of ``'text'``, ``'html'``, ``'text'``
+                          or ``'xhtml'``.  Default is ``'text'``.
+    :param links: additional links.  Must be a list of dictionaries with
+                  href (required) and rel, type, hreflang, title, length
+                  (all optional)
+    :param generator: the software that generated this feed.  This must be
+                      a tuple in the form ``(name, url, version)``.  If
+                      you don't want to specify one of them, set the item
+                      to `None`.
+    :param entries: a list with the entries for the feed. Entries can also
+                    be added later with :meth:`add`.
+
+    For more information on the elements see
+    http://www.atomenabled.org/developers/syndication/
+
+    Everywhere where a list is demanded, any iterable can be used.
+    """
+
+    default_generator = ("Werkzeug", None, None)
+
+    def __init__(self, title=None, entries=None, **kwargs):
+        self.title = title
+        self.title_type = kwargs.get("title_type", "text")
+        self.url = kwargs.get("url")
+        self.feed_url = kwargs.get("feed_url", self.url)
+        self.id = kwargs.get("id", self.feed_url)
+        self.updated = kwargs.get("updated")
+        self.author = kwargs.get("author", ())
+        self.icon = kwargs.get("icon")
+        self.logo = kwargs.get("logo")
+        self.rights = kwargs.get("rights")
+        self.rights_type = kwargs.get("rights_type")
+        self.subtitle = kwargs.get("subtitle")
+        self.subtitle_type = kwargs.get("subtitle_type", "text")
+        self.generator = kwargs.get("generator")
+        if self.generator is None:
+            self.generator = self.default_generator
+        self.links = kwargs.get("links", [])
+        self.entries = list(entries) if entries else []
+
+        if not hasattr(self.author, "__iter__") or isinstance(
+            self.author, string_types + (dict,)
+        ):
+            self.author = [self.author]
+        for i, author in enumerate(self.author):
+            if not isinstance(author, dict):
+                self.author[i] = {"name": author}
+
+        if not self.title:
+            raise ValueError("title is required")
+        if not self.id:
+            raise ValueError("id is required")
+        for author in self.author:
+            if "name" not in author:
+                raise TypeError("author must contain at least a name")
+
+    def add(self, *args, **kwargs):
+        """Add a new entry to the feed.  This function can either be called
+        with a :class:`FeedEntry` or some keyword and positional arguments
+        that are forwarded to the :class:`FeedEntry` constructor.
+        """
+        if len(args) == 1 and not kwargs and isinstance(args[0], FeedEntry):
+            self.entries.append(args[0])
+        else:
+            kwargs["feed_url"] = self.feed_url
+            self.entries.append(FeedEntry(*args, **kwargs))
+
+    def __repr__(self):
+        return "<%s %r (%d entries)>" % (
+            self.__class__.__name__,
+            self.title,
+            len(self.entries),
+        )
+
+    def generate(self):
+        """Return a generator that yields pieces of XML."""
+        # atom demands either an author element in every entry or a global one
+        if not self.author:
+            if any(not e.author for e in self.entries):
+                self.author = ({"name": "Unknown author"},)
+
+        if not self.updated:
+            dates = sorted([entry.updated for entry in self.entries])
+            self.updated = dates[-1] if dates else datetime.utcnow()
+
+        yield u'<?xml version="1.0" encoding="utf-8"?>\n'
+        yield u'<feed xmlns="http://www.w3.org/2005/Atom">\n'
+        yield "  " + _make_text_block("title", self.title, self.title_type)
+        yield u"  <id>%s</id>\n" % escape(self.id)
+        yield u"  <updated>%s</updated>\n" % format_iso8601(self.updated)
+        if self.url:
+            yield u'  <link href="%s" />\n' % escape(self.url)
+        if self.feed_url:
+            yield u'  <link href="%s" rel="self" />\n' % escape(self.feed_url)
+        for link in self.links:
+            yield u"  <link %s/>\n" % "".join(
+                '%s="%s" ' % (k, escape(link[k])) for k in link
+            )
+        for author in self.author:
+            yield u"  <author>\n"
+            yield u"    <name>%s</name>\n" % escape(author["name"])
+            if "uri" in author:
+                yield u"    <uri>%s</uri>\n" % escape(author["uri"])
+            if "email" in author:
+                yield "    <email>%s</email>\n" % escape(author["email"])
+            yield "  </author>\n"
+        if self.subtitle:
+            yield "  " + _make_text_block("subtitle", self.subtitle, self.subtitle_type)
+        if self.icon:
+            yield u"  <icon>%s</icon>\n" % escape(self.icon)
+        if self.logo:
+            yield u"  <logo>%s</logo>\n" % escape(self.logo)
+        if self.rights:
+            yield "  " + _make_text_block("rights", self.rights, self.rights_type)
+        generator_name, generator_url, generator_version = self.generator
+        if generator_name or generator_url or generator_version:
+            tmp = [u"  <generator"]
+            if generator_url:
+                tmp.append(u' uri="%s"' % escape(generator_url))
+            if generator_version:
+                tmp.append(u' version="%s"' % escape(generator_version))
+            tmp.append(u">%s</generator>\n" % escape(generator_name))
+            yield u"".join(tmp)
+        for entry in self.entries:
+            for line in entry.generate():
+                yield u"  " + line
+        yield u"</feed>\n"
+
+    def to_string(self):
+        """Convert the feed into a string."""
+        return u"".join(self.generate())
+
+    def get_response(self):
+        """Return a response object for the feed."""
+        return BaseResponse(self.to_string(), mimetype="application/atom+xml")
+
+    def __call__(self, environ, start_response):
+        """Use the class as WSGI response object."""
+        return self.get_response()(environ, start_response)
+
+    def __str__(self):
+        return self.to_string()
+
+
+@implements_to_string
+class FeedEntry(object):
+
+    """Represents a single entry in a feed.
+
+    :param title: the title of the entry. Required.
+    :param title_type: the type attribute for the title element.  One of
+                       ``'html'``, ``'text'`` or ``'xhtml'``.
+    :param content: the content of the entry.
+    :param content_type: the type attribute for the content element.  One
+                         of ``'html'``, ``'text'`` or ``'xhtml'``.
+    :param summary: a summary of the entry's content.
+    :param summary_type: the type attribute for the summary element.  One
+                         of ``'html'``, ``'text'`` or ``'xhtml'``.
+    :param url: the url for the entry.
+    :param id: a globally unique id for the entry.  Must be an URI.  If
+               not present the URL is used, but one of both is required.
+    :param updated: the time the entry was modified the last time.  Must
+                    be a :class:`datetime.datetime` object.  Treated as
+                    UTC if naive datetime. Required.
+    :param author: the author of the entry.  Must be either a string (the
+                   name) or a dict with name (required) and uri or
+                   email (both optional).  Can be a list of (may be
+                   mixed, too) strings and dicts, too, if there are
+                   multiple authors. Required if the feed does not have an
+                   author element.
+    :param published: the time the entry was initially published.  Must
+                      be a :class:`datetime.datetime` object.  Treated as
+                      UTC if naive datetime.
+    :param rights: copyright information for the entry.
+    :param rights_type: the type attribute for the rights element.  One of
+                        ``'html'``, ``'text'`` or ``'xhtml'``.  Default is
+                        ``'text'``.
+    :param links: additional links.  Must be a list of dictionaries with
+                  href (required) and rel, type, hreflang, title, length
+                  (all optional)
+    :param categories: categories for the entry. Must be a list of dictionaries
+                       with term (required), scheme and label (all optional)
+    :param xml_base: The xml base (url) for this feed item.  If not provided
+                     it will default to the item url.
+
+    For more information on the elements see
+    http://www.atomenabled.org/developers/syndication/
+
+    Everywhere where a list is demanded, any iterable can be used.
+    """
+
+    def __init__(self, title=None, content=None, feed_url=None, **kwargs):
+        self.title = title
+        self.title_type = kwargs.get("title_type", "text")
+        self.content = content
+        self.content_type = kwargs.get("content_type", "html")
+        self.url = kwargs.get("url")
+        self.id = kwargs.get("id", self.url)
+        self.updated = kwargs.get("updated")
+        self.summary = kwargs.get("summary")
+        self.summary_type = kwargs.get("summary_type", "html")
+        self.author = kwargs.get("author", ())
+        self.published = kwargs.get("published")
+        self.rights = kwargs.get("rights")
+        self.links = kwargs.get("links", [])
+        self.categories = kwargs.get("categories", [])
+        self.xml_base = kwargs.get("xml_base", feed_url)
+
+        if not hasattr(self.author, "__iter__") or isinstance(
+            self.author, string_types + (dict,)
+        ):
+            self.author = [self.author]
+        for i, author in enumerate(self.author):
+            if not isinstance(author, dict):
+                self.author[i] = {"name": author}
+
+        if not self.title:
+            raise ValueError("title is required")
+        if not self.id:
+            raise ValueError("id is required")
+        if not self.updated:
+            raise ValueError("updated is required")
+
+    def __repr__(self):
+        return "<%s %r>" % (self.__class__.__name__, self.title)
+
+    def generate(self):
+        """Yields pieces of ATOM XML."""
+        base = ""
+        if self.xml_base:
+            base = ' xml:base="%s"' % escape(self.xml_base)
+        yield u"<entry%s>\n" % base
+        yield u"  " + _make_text_block("title", self.title, self.title_type)
+        yield u"  <id>%s</id>\n" % escape(self.id)
+        yield u"  <updated>%s</updated>\n" % format_iso8601(self.updated)
+        if self.published:
+            yield u"  <published>%s</published>\n" % format_iso8601(self.published)
+        if self.url:
+            yield u'  <link href="%s" />\n' % escape(self.url)
+        for author in self.author:
+            yield u"  <author>\n"
+            yield u"    <name>%s</name>\n" % escape(author["name"])
+            if "uri" in author:
+                yield u"    <uri>%s</uri>\n" % escape(author["uri"])
+            if "email" in author:
+                yield u"    <email>%s</email>\n" % escape(author["email"])
+            yield u"  </author>\n"
+        for link in self.links:
+            yield u"  <link %s/>\n" % "".join(
+                '%s="%s" ' % (k, escape(link[k])) for k in link
+            )
+        for category in self.categories:
+            yield u"  <category %s/>\n" % "".join(
+                '%s="%s" ' % (k, escape(category[k])) for k in category
+            )
+        if self.summary:
+            yield u"  " + _make_text_block("summary", self.summary, self.summary_type)
+        if self.content:
+            yield u"  " + _make_text_block("content", self.content, self.content_type)
+        yield u"</entry>\n"
+
+    def to_string(self):
+        """Convert the feed item into a unicode object."""
+        return u"".join(self.generate())
+
+    def __str__(self):
+        return self.to_string()

python/werkzeug/contrib/cache.py

@@ -0,0 +1,933 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.contrib.cache
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    The main problem with dynamic Web sites is, well, they're dynamic.  Each
+    time a user requests a page, the webserver executes a lot of code, queries
+    the database, renders templates until the visitor gets the page he sees.
+
+    This is a lot more expensive than just loading a file from the file system
+    and sending it to the visitor.
+
+    For most Web applications, this overhead isn't a big deal but once it
+    becomes, you will be glad to have a cache system in place.
+
+    How Caching Works
+    =================
+
+    Caching is pretty simple.  Basically you have a cache object lurking around
+    somewhere that is connected to a remote cache or the file system or
+    something else.  When the request comes in you check if the current page
+    is already in the cache and if so, you're returning it from the cache.
+    Otherwise you generate the page and put it into the cache. (Or a fragment
+    of the page, you don't have to cache the full thing)
+
+    Here is a simple example of how to cache a sidebar for 5 minutes::
+
+        def get_sidebar(user):
+            identifier = 'sidebar_for/user%d' % user.id
+            value = cache.get(identifier)
+            if value is not None:
+                return value
+            value = generate_sidebar_for(user=user)
+            cache.set(identifier, value, timeout=60 * 5)
+            return value
+
+    Creating a Cache Object
+    =======================
+
+    To create a cache object you just import the cache system of your choice
+    from the cache module and instantiate it.  Then you can start working
+    with that object:
+
+    >>> from werkzeug.contrib.cache import SimpleCache
+    >>> c = SimpleCache()
+    >>> c.set("foo", "value")
+    >>> c.get("foo")
+    'value'
+    >>> c.get("missing") is None
+    True
+
+    Please keep in mind that you have to create the cache and put it somewhere
+    you have access to it (either as a module global you can import or you just
+    put it into your WSGI application).
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import errno
+import os
+import platform
+import re
+import tempfile
+import warnings
+from hashlib import md5
+from time import time
+
+from .._compat import integer_types
+from .._compat import iteritems
+from .._compat import string_types
+from .._compat import text_type
+from .._compat import to_native
+from ..posixemulation import rename
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: no cover
+    import pickle
+
+warnings.warn(
+    "'werkzeug.contrib.cache' is deprecated as of version 0.15 and will"
+    " be removed in version 1.0. It has moved to https://github.com"
+    "/pallets/cachelib.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
+def _items(mappingorseq):
+    """Wrapper for efficient iteration over mappings represented by dicts
+    or sequences::
+
+        >>> for k, v in _items((i, i*i) for i in xrange(5)):
+        ...    assert k*k == v
+
+        >>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
+        ...    assert k*k == v
+
+    """
+    if hasattr(mappingorseq, "items"):
+        return iteritems(mappingorseq)
+    return mappingorseq
+
+
+class BaseCache(object):
+    """Baseclass for the cache systems.  All the cache systems implement this
+    API or a superset of it.
+
+    :param default_timeout: the default timeout (in seconds) that is used if
+                            no timeout is specified on :meth:`set`. A timeout
+                            of 0 indicates that the cache never expires.
+    """
+
+    def __init__(self, default_timeout=300):
+        self.default_timeout = default_timeout
+
+    def _normalize_timeout(self, timeout):
+        if timeout is None:
+            timeout = self.default_timeout
+        return timeout
+
+    def get(self, key):
+        """Look up key in the cache and return the value for it.
+
+        :param key: the key to be looked up.
+        :returns: The value if it exists and is readable, else ``None``.
+        """
+        return None
+
+    def delete(self, key):
+        """Delete `key` from the cache.
+
+        :param key: the key to delete.
+        :returns: Whether the key existed and has been deleted.
+        :rtype: boolean
+        """
+        return True
+
+    def get_many(self, *keys):
+        """Returns a list of values for the given keys.
+        For each key an item in the list is created::
+
+            foo, bar = cache.get_many("foo", "bar")
+
+        Has the same error handling as :meth:`get`.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
+        return [self.get(k) for k in keys]
+
+    def get_dict(self, *keys):
+        """Like :meth:`get_many` but return a dict::
+
+            d = cache.get_dict("foo", "bar")
+            foo = d["foo"]
+            bar = d["bar"]
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
+        return dict(zip(keys, self.get_many(*keys)))
+
+    def set(self, key, value, timeout=None):
+        """Add a new key/value to the cache (overwrites value, if key already
+        exists in the cache).
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 idicates that the cache never expires.
+        :returns: ``True`` if key has been updated, ``False`` for backend
+                  errors. Pickling errors, however, will raise a subclass of
+                  ``pickle.PickleError``.
+        :rtype: boolean
+        """
+        return True
+
+    def add(self, key, value, timeout=None):
+        """Works like :meth:`set` but does not overwrite the values of already
+        existing keys.
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 idicates that the cache never expires.
+        :returns: Same as :meth:`set`, but also ``False`` for already
+                  existing keys.
+        :rtype: boolean
+        """
+        return True
+
+    def set_many(self, mapping, timeout=None):
+        """Sets multiple keys and values from a mapping.
+
+        :param mapping: a mapping with the keys/values to set.
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 idicates that the cache never expires.
+        :returns: Whether all given keys have been set.
+        :rtype: boolean
+        """
+        rv = True
+        for key, value in _items(mapping):
+            if not self.set(key, value, timeout):
+                rv = False
+        return rv
+
+    def delete_many(self, *keys):
+        """Deletes multiple keys at once.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        :returns: Whether all given keys have been deleted.
+        :rtype: boolean
+        """
+        return all(self.delete(key) for key in keys)
+
+    def has(self, key):
+        """Checks if a key exists in the cache without returning it. This is a
+        cheap operation that bypasses loading the actual data on the backend.
+
+        This method is optional and may not be implemented on all caches.
+
+        :param key: the key to check
+        """
+        raise NotImplementedError(
+            "%s doesn't have an efficient implementation of `has`. That "
+            "means it is impossible to check whether a key exists without "
+            "fully loading the key's data. Consider using `self.get` "
+            "explicitly if you don't care about performance."
+        )
+
+    def clear(self):
+        """Clears the cache.  Keep in mind that not all caches support
+        completely clearing the cache.
+
+        :returns: Whether the cache has been cleared.
+        :rtype: boolean
+        """
+        return True
+
+    def inc(self, key, delta=1):
+        """Increments the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to add.
+        :returns: The new value or ``None`` for backend errors.
+        """
+        value = (self.get(key) or 0) + delta
+        return value if self.set(key, value) else None
+
+    def dec(self, key, delta=1):
+        """Decrements the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `-delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to subtract.
+        :returns: The new value or `None` for backend errors.
+        """
+        value = (self.get(key) or 0) - delta
+        return value if self.set(key, value) else None
+
+
+class NullCache(BaseCache):
+    """A cache that doesn't cache.  This can be useful for unit testing.
+
+    :param default_timeout: a dummy parameter that is ignored but exists
+                            for API compatibility with other caches.
+    """
+
+    def has(self, key):
+        return False
+
+
+class SimpleCache(BaseCache):
+    """Simple memory cache for single process environments.  This class exists
+    mainly for the development server and is not 100% thread safe.  It tries
+    to use as many atomic operations as possible and no locks for simplicity
+    but it could happen under heavy load that keys are added multiple times.
+
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    """
+
+    def __init__(self, threshold=500, default_timeout=300):
+        BaseCache.__init__(self, default_timeout)
+        self._cache = {}
+        self.clear = self._cache.clear
+        self._threshold = threshold
+
+    def _prune(self):
+        if len(self._cache) > self._threshold:
+            now = time()
+            toremove = []
+            for idx, (key, (expires, _)) in enumerate(self._cache.items()):
+                if (expires != 0 and expires <= now) or idx % 3 == 0:
+                    toremove.append(key)
+            for key in toremove:
+                self._cache.pop(key, None)
+
+    def _normalize_timeout(self, timeout):
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout > 0:
+            timeout = time() + timeout
+        return timeout
+
+    def get(self, key):
+        try:
+            expires, value = self._cache[key]
+            if expires == 0 or expires > time():
+                return pickle.loads(value)
+        except (KeyError, pickle.PickleError):
+            return None
+
+    def set(self, key, value, timeout=None):
+        expires = self._normalize_timeout(timeout)
+        self._prune()
+        self._cache[key] = (expires, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
+        return True
+
+    def add(self, key, value, timeout=None):
+        expires = self._normalize_timeout(timeout)
+        self._prune()
+        item = (expires, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
+        if key in self._cache:
+            return False
+        self._cache.setdefault(key, item)
+        return True
+
+    def delete(self, key):
+        return self._cache.pop(key, None) is not None
+
+    def has(self, key):
+        try:
+            expires, value = self._cache[key]
+            return expires == 0 or expires > time()
+        except KeyError:
+            return False
+
+
+_test_memcached_key = re.compile(r"[^\x00-\x21\xff]{1,250}$").match
+
+
+class MemcachedCache(BaseCache):
+    """A cache that uses memcached as backend.
+
+    The first argument can either be an object that resembles the API of a
+    :class:`memcache.Client` or a tuple/list of server addresses. In the
+    event that a tuple/list is passed, Werkzeug tries to import the best
+    available memcache library.
+
+    This cache looks into the following packages/modules to find bindings for
+    memcached:
+
+        - ``pylibmc``
+        - ``google.appengine.api.memcached``
+        - ``memcached``
+        - ``libmc``
+
+    Implementation notes:  This cache backend works around some limitations in
+    memcached to simplify the interface.  For example unicode keys are encoded
+    to utf-8 on the fly.  Methods such as :meth:`~BaseCache.get_dict` return
+    the keys in the same format as passed.  Furthermore all get methods
+    silently ignore key errors to not cause problems when untrusted user data
+    is passed to the get methods which is often the case in web applications.
+
+    :param servers: a list or tuple of server addresses or alternatively
+                    a :class:`memcache.Client` or a compatible client.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param key_prefix: a prefix that is added before all keys.  This makes it
+                       possible to use the same memcached server for different
+                       applications.  Keep in mind that
+                       :meth:`~BaseCache.clear` will also clear keys with a
+                       different prefix.
+    """
+
+    def __init__(self, servers=None, default_timeout=300, key_prefix=None):
+        BaseCache.__init__(self, default_timeout)
+        if servers is None or isinstance(servers, (list, tuple)):
+            if servers is None:
+                servers = ["127.0.0.1:11211"]
+            self._client = self.import_preferred_memcache_lib(servers)
+            if self._client is None:
+                raise RuntimeError("no memcache module found")
+        else:
+            # NOTE: servers is actually an already initialized memcache
+            # client.
+            self._client = servers
+
+        self.key_prefix = to_native(key_prefix)
+
+    def _normalize_key(self, key):
+        key = to_native(key, "utf-8")
+        if self.key_prefix:
+            key = self.key_prefix + key
+        return key
+
+    def _normalize_timeout(self, timeout):
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout > 0:
+            timeout = int(time()) + timeout
+        return timeout
+
+    def get(self, key):
+        key = self._normalize_key(key)
+        # memcached doesn't support keys longer than that.  Because often
+        # checks for so long keys can occur because it's tested from user
+        # submitted data etc we fail silently for getting.
+        if _test_memcached_key(key):
+            return self._client.get(key)
+
+    def get_dict(self, *keys):
+        key_mapping = {}
+        have_encoded_keys = False
+        for key in keys:
+            encoded_key = self._normalize_key(key)
+            if not isinstance(key, str):
+                have_encoded_keys = True
+            if _test_memcached_key(key):
+                key_mapping[encoded_key] = key
+        _keys = list(key_mapping)
+        d = rv = self._client.get_multi(_keys)
+        if have_encoded_keys or self.key_prefix:
+            rv = {}
+            for key, value in iteritems(d):
+                rv[key_mapping[key]] = value
+        if len(rv) < len(keys):
+            for key in keys:
+                if key not in rv:
+                    rv[key] = None
+        return rv
+
+    def add(self, key, value, timeout=None):
+        key = self._normalize_key(key)
+        timeout = self._normalize_timeout(timeout)
+        return self._client.add(key, value, timeout)
+
+    def set(self, key, value, timeout=None):
+        key = self._normalize_key(key)
+        timeout = self._normalize_timeout(timeout)
+        return self._client.set(key, value, timeout)
+
+    def get_many(self, *keys):
+        d = self.get_dict(*keys)
+        return [d[key] for key in keys]
+
+    def set_many(self, mapping, timeout=None):
+        new_mapping = {}
+        for key, value in _items(mapping):
+            key = self._normalize_key(key)
+            new_mapping[key] = value
+
+        timeout = self._normalize_timeout(timeout)
+        failed_keys = self._client.set_multi(new_mapping, timeout)
+        return not failed_keys
+
+    def delete(self, key):
+        key = self._normalize_key(key)
+        if _test_memcached_key(key):
+            return self._client.delete(key)
+
+    def delete_many(self, *keys):
+        new_keys = []
+        for key in keys:
+            key = self._normalize_key(key)
+            if _test_memcached_key(key):
+                new_keys.append(key)
+        return self._client.delete_multi(new_keys)
+
+    def has(self, key):
+        key = self._normalize_key(key)
+        if _test_memcached_key(key):
+            return self._client.append(key, "")
+        return False
+
+    def clear(self):
+        return self._client.flush_all()
+
+    def inc(self, key, delta=1):
+        key = self._normalize_key(key)
+        return self._client.incr(key, delta)
+
+    def dec(self, key, delta=1):
+        key = self._normalize_key(key)
+        return self._client.decr(key, delta)
+
+    def import_preferred_memcache_lib(self, servers):
+        """Returns an initialized memcache client.  Used by the constructor."""
+        try:
+            import pylibmc
+        except ImportError:
+            pass
+        else:
+            return pylibmc.Client(servers)
+
+        try:
+            from google.appengine.api import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client()
+
+        try:
+            import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client(servers)
+
+        try:
+            import libmc
+        except ImportError:
+            pass
+        else:
+            return libmc.Client(servers)
+
+
+# backwards compatibility
+GAEMemcachedCache = MemcachedCache
+
+
+class RedisCache(BaseCache):
+    """Uses the Redis key-value store as a cache backend.
+
+    The first argument can be either a string denoting address of the Redis
+    server or an object resembling an instance of a redis.Redis class.
+
+    Note: Python Redis API already takes care of encoding unicode strings on
+    the fly.
+
+    .. versionadded:: 0.7
+
+    .. versionadded:: 0.8
+       `key_prefix` was added.
+
+    .. versionchanged:: 0.8
+       This cache backend now properly serializes objects.
+
+    .. versionchanged:: 0.8.3
+       This cache backend now supports password authentication.
+
+    .. versionchanged:: 0.10
+        ``**kwargs`` is now passed to the redis object.
+
+    :param host: address of the Redis server or an object which API is
+                 compatible with the official Python Redis client (redis-py).
+    :param port: port number on which Redis server listens for connections.
+    :param password: password authentication for the Redis server.
+    :param db: db (zero-based numeric index) on Redis Server to connect.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param key_prefix: A prefix that should be added to all keys.
+
+    Any additional keyword arguments will be passed to ``redis.Redis``.
+    """
+
+    def __init__(
+        self,
+        host="localhost",
+        port=6379,
+        password=None,
+        db=0,
+        default_timeout=300,
+        key_prefix=None,
+        **kwargs
+    ):
+        BaseCache.__init__(self, default_timeout)
+        if host is None:
+            raise ValueError("RedisCache host parameter may not be None")
+        if isinstance(host, string_types):
+            try:
+                import redis
+            except ImportError:
+                raise RuntimeError("no redis module found")
+            if kwargs.get("decode_responses", None):
+                raise ValueError("decode_responses is not supported by RedisCache.")
+            self._client = redis.Redis(
+                host=host, port=port, password=password, db=db, **kwargs
+            )
+        else:
+            self._client = host
+        self.key_prefix = key_prefix or ""
+
+    def _normalize_timeout(self, timeout):
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout == 0:
+            timeout = -1
+        return timeout
+
+    def dump_object(self, value):
+        """Dumps an object into a string for redis.  By default it serializes
+        integers as regular string and pickle dumps everything else.
+        """
+        t = type(value)
+        if t in integer_types:
+            return str(value).encode("ascii")
+        return b"!" + pickle.dumps(value)
+
+    def load_object(self, value):
+        """The reversal of :meth:`dump_object`.  This might be called with
+        None.
+        """
+        if value is None:
+            return None
+        if value.startswith(b"!"):
+            try:
+                return pickle.loads(value[1:])
+            except pickle.PickleError:
+                return None
+        try:
+            return int(value)
+        except ValueError:
+            # before 0.8 we did not have serialization.  Still support that.
+            return value
+
+    def get(self, key):
+        return self.load_object(self._client.get(self.key_prefix + key))
+
+    def get_many(self, *keys):
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
+        return [self.load_object(x) for x in self._client.mget(keys)]
+
+    def set(self, key, value, timeout=None):
+        timeout = self._normalize_timeout(timeout)
+        dump = self.dump_object(value)
+        if timeout == -1:
+            result = self._client.set(name=self.key_prefix + key, value=dump)
+        else:
+            result = self._client.setex(
+                name=self.key_prefix + key, value=dump, time=timeout
+            )
+        return result
+
+    def add(self, key, value, timeout=None):
+        timeout = self._normalize_timeout(timeout)
+        dump = self.dump_object(value)
+        return self._client.setnx(
+            name=self.key_prefix + key, value=dump
+        ) and self._client.expire(name=self.key_prefix + key, time=timeout)
+
+    def set_many(self, mapping, timeout=None):
+        timeout = self._normalize_timeout(timeout)
+        # Use transaction=False to batch without calling redis MULTI
+        # which is not supported by twemproxy
+        pipe = self._client.pipeline(transaction=False)
+
+        for key, value in _items(mapping):
+            dump = self.dump_object(value)
+            if timeout == -1:
+                pipe.set(name=self.key_prefix + key, value=dump)
+            else:
+                pipe.setex(name=self.key_prefix + key, value=dump, time=timeout)
+        return pipe.execute()
+
+    def delete(self, key):
+        return self._client.delete(self.key_prefix + key)
+
+    def delete_many(self, *keys):
+        if not keys:
+            return
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
+        return self._client.delete(*keys)
+
+    def has(self, key):
+        return self._client.exists(self.key_prefix + key)
+
+    def clear(self):
+        status = False
+        if self.key_prefix:
+            keys = self._client.keys(self.key_prefix + "*")
+            if keys:
+                status = self._client.delete(*keys)
+        else:
+            status = self._client.flushdb()
+        return status
+
+    def inc(self, key, delta=1):
+        return self._client.incr(name=self.key_prefix + key, amount=delta)
+
+    def dec(self, key, delta=1):
+        return self._client.decr(name=self.key_prefix + key, amount=delta)
+
+
+class FileSystemCache(BaseCache):
+    """A cache that stores the items on the file system.  This cache depends
+    on being the only user of the `cache_dir`.  Make absolutely sure that
+    nobody but this cache stores files there or otherwise the cache will
+    randomly delete files therein.
+
+    :param cache_dir: the directory where cache files are stored.
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some. A threshold value of 0
+                      indicates no threshold.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param mode: the file mode wanted for the cache files, default 0600
+    """
+
+    #: used for temporary files by the FileSystemCache
+    _fs_transaction_suffix = ".__wz_cache"
+    #: keep amount of files in a cache element
+    _fs_count_file = "__wz_cache_count"
+
+    def __init__(self, cache_dir, threshold=500, default_timeout=300, mode=0o600):
+        BaseCache.__init__(self, default_timeout)
+        self._path = cache_dir
+        self._threshold = threshold
+        self._mode = mode
+
+        try:
+            os.makedirs(self._path)
+        except OSError as ex:
+            if ex.errno != errno.EEXIST:
+                raise
+
+        self._update_count(value=len(self._list_dir()))
+
+    @property
+    def _file_count(self):
+        return self.get(self._fs_count_file) or 0
+
+    def _update_count(self, delta=None, value=None):
+        # If we have no threshold, don't count files
+        if self._threshold == 0:
+            return
+
+        if delta:
+            new_count = self._file_count + delta
+        else:
+            new_count = value or 0
+        self.set(self._fs_count_file, new_count, mgmt_element=True)
+
+    def _normalize_timeout(self, timeout):
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout != 0:
+            timeout = time() + timeout
+        return int(timeout)
+
+    def _list_dir(self):
+        """return a list of (fully qualified) cache filenames
+        """
+        mgmt_files = [
+            self._get_filename(name).split("/")[-1] for name in (self._fs_count_file,)
+        ]
+        return [
+            os.path.join(self._path, fn)
+            for fn in os.listdir(self._path)
+            if not fn.endswith(self._fs_transaction_suffix) and fn not in mgmt_files
+        ]
+
+    def _prune(self):
+        if self._threshold == 0 or not self._file_count > self._threshold:
+            return
+
+        entries = self._list_dir()
+        now = time()
+        for idx, fname in enumerate(entries):
+            try:
+                remove = False
+                with open(fname, "rb") as f:
+                    expires = pickle.load(f)
+                remove = (expires != 0 and expires <= now) or idx % 3 == 0
+
+                if remove:
+                    os.remove(fname)
+            except (IOError, OSError):
+                pass
+        self._update_count(value=len(self._list_dir()))
+
+    def clear(self):
+        for fname in self._list_dir():
+            try:
+                os.remove(fname)
+            except (IOError, OSError):
+                self._update_count(value=len(self._list_dir()))
+                return False
+        self._update_count(value=0)
+        return True
+
+    def _get_filename(self, key):
+        if isinstance(key, text_type):
+            key = key.encode("utf-8")  # XXX unicode review
+        hash = md5(key).hexdigest()
+        return os.path.join(self._path, hash)
+
+    def get(self, key):
+        filename = self._get_filename(key)
+        try:
+            with open(filename, "rb") as f:
+                pickle_time = pickle.load(f)
+                if pickle_time == 0 or pickle_time >= time():
+                    return pickle.load(f)
+                else:
+                    os.remove(filename)
+                    return None
+        except (IOError, OSError, pickle.PickleError):
+            return None
+
+    def add(self, key, value, timeout=None):
+        filename = self._get_filename(key)
+        if not os.path.exists(filename):
+            return self.set(key, value, timeout)
+        return False
+
+    def set(self, key, value, timeout=None, mgmt_element=False):
+        # Management elements have no timeout
+        if mgmt_element:
+            timeout = 0
+
+        # Don't prune on management element update, to avoid loop
+        else:
+            self._prune()
+
+        timeout = self._normalize_timeout(timeout)
+        filename = self._get_filename(key)
+        try:
+            fd, tmp = tempfile.mkstemp(
+                suffix=self._fs_transaction_suffix, dir=self._path
+            )
+            with os.fdopen(fd, "wb") as f:
+                pickle.dump(timeout, f, 1)
+                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
+            rename(tmp, filename)
+            os.chmod(filename, self._mode)
+        except (IOError, OSError):
+            return False
+        else:
+            # Management elements should not count towards threshold
+            if not mgmt_element:
+                self._update_count(delta=1)
+            return True
+
+    def delete(self, key, mgmt_element=False):
+        try:
+            os.remove(self._get_filename(key))
+        except (IOError, OSError):
+            return False
+        else:
+            # Management elements should not count towards threshold
+            if not mgmt_element:
+                self._update_count(delta=-1)
+            return True
+
+    def has(self, key):
+        filename = self._get_filename(key)
+        try:
+            with open(filename, "rb") as f:
+                pickle_time = pickle.load(f)
+                if pickle_time == 0 or pickle_time >= time():
+                    return True
+                else:
+                    os.remove(filename)
+                    return False
+        except (IOError, OSError, pickle.PickleError):
+            return False
+
+
+class UWSGICache(BaseCache):
+    """Implements the cache using uWSGI's caching framework.
+
+    .. note::
+        This class cannot be used when running under PyPy, because the uWSGI
+        API implementation for PyPy is lacking the needed functionality.
+
+    :param default_timeout: The default timeout in seconds.
+    :param cache: The name of the caching instance to connect to, for
+        example: mycache@localhost:3031, defaults to an empty string, which
+        means uWSGI will cache in the local instance. If the cache is in the
+        same instance as the werkzeug app, you only have to provide the name of
+        the cache.
+    """
+
+    def __init__(self, default_timeout=300, cache=""):
+        BaseCache.__init__(self, default_timeout)
+
+        if platform.python_implementation() == "PyPy":
+            raise RuntimeError(
+                "uWSGI caching does not work under PyPy, see "
+                "the docs for more details."
+            )
+
+        try:
+            import uwsgi
+
+            self._uwsgi = uwsgi
+        except ImportError:
+            raise RuntimeError(
+                "uWSGI could not be imported, are you running under uWSGI?"
+            )
+
+        self.cache = cache
+
+    def get(self, key):
+        rv = self._uwsgi.cache_get(key, self.cache)
+        if rv is None:
+            return
+        return pickle.loads(rv)
+
+    def delete(self, key):
+        return self._uwsgi.cache_del(key, self.cache)
+
+    def set(self, key, value, timeout=None):
+        return self._uwsgi.cache_update(
+            key, pickle.dumps(value), self._normalize_timeout(timeout), self.cache
+        )
+
+    def add(self, key, value, timeout=None):
+        return self._uwsgi.cache_set(
+            key, pickle.dumps(value), self._normalize_timeout(timeout), self.cache
+        )
+
+    def clear(self):
+        return self._uwsgi.cache_clear(self.cache)
+
+    def has(self, key):
+        return self._uwsgi.cache_exists(key, self.cache) is not None

python/werkzeug/contrib/fixers.py

@@ -0,0 +1,262 @@
+"""
+Fixers
+======
+
+.. warning::
+    .. deprecated:: 0.15
+        ``ProxyFix`` has moved to :mod:`werkzeug.middleware.proxy_fix`.
+        All other code in this module is deprecated and will be removed
+        in version 1.0.
+
+.. versionadded:: 0.5
+
+This module includes various helpers that fix web server behavior.
+
+.. autoclass:: ProxyFix
+   :members:
+
+.. autoclass:: CGIRootFix
+
+.. autoclass:: PathInfoFromRequestUriFix
+
+.. autoclass:: HeaderRewriterFix
+
+.. autoclass:: InternetExplorerFix
+
+:copyright: 2007 Pallets
+:license: BSD-3-Clause
+"""
+import warnings
+
+from ..datastructures import Headers
+from ..datastructures import ResponseCacheControl
+from ..http import parse_cache_control_header
+from ..http import parse_options_header
+from ..http import parse_set_header
+from ..middleware.proxy_fix import ProxyFix as _ProxyFix
+from ..useragents import UserAgent
+
+try:
+    from urllib.parse import unquote
+except ImportError:
+    from urllib import unquote
+
+
+class CGIRootFix(object):
+    """Wrap the application in this middleware if you are using FastCGI
+    or CGI and you have problems with your app root being set to the CGI
+    script's path instead of the path users are going to visit.
+
+    :param app: the WSGI application
+    :param app_root: Defaulting to ``'/'``, you can set this to
+        something else if your app is mounted somewhere else.
+
+    .. deprecated:: 0.15
+        This middleware will be removed in version 1.0.
+
+    .. versionchanged:: 0.9
+        Added `app_root` parameter and renamed from
+        ``LighttpdCGIRootFix``.
+    """
+
+    def __init__(self, app, app_root="/"):
+        warnings.warn(
+            "'CGIRootFix' is deprecated as of version 0.15 and will be"
+            " removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.app = app
+        self.app_root = app_root.strip("/")
+
+    def __call__(self, environ, start_response):
+        environ["SCRIPT_NAME"] = self.app_root
+        return self.app(environ, start_response)
+
+
+class LighttpdCGIRootFix(CGIRootFix):
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "'LighttpdCGIRootFix' is renamed 'CGIRootFix'. Both will be"
+            " removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super(LighttpdCGIRootFix, self).__init__(*args, **kwargs)
+
+
+class PathInfoFromRequestUriFix(object):
+    """On windows environment variables are limited to the system charset
+    which makes it impossible to store the `PATH_INFO` variable in the
+    environment without loss of information on some systems.
+
+    This is for example a problem for CGI scripts on a Windows Apache.
+
+    This fixer works by recreating the `PATH_INFO` from `REQUEST_URI`,
+    `REQUEST_URL`, or `UNENCODED_URL` (whatever is available).  Thus the
+    fix can only be applied if the webserver supports either of these
+    variables.
+
+    :param app: the WSGI application
+
+    .. deprecated:: 0.15
+        This middleware will be removed in version 1.0.
+    """
+
+    def __init__(self, app):
+        warnings.warn(
+            "'PathInfoFromRequestUriFix' is deprecated as of version"
+            " 0.15 and will be removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        for key in "REQUEST_URL", "REQUEST_URI", "UNENCODED_URL":
+            if key not in environ:
+                continue
+            request_uri = unquote(environ[key])
+            script_name = unquote(environ.get("SCRIPT_NAME", ""))
+            if request_uri.startswith(script_name):
+                environ["PATH_INFO"] = request_uri[len(script_name) :].split("?", 1)[0]
+                break
+        return self.app(environ, start_response)
+
+
+class ProxyFix(_ProxyFix):
+    """
+    .. deprecated:: 0.15
+        ``werkzeug.contrib.fixers.ProxyFix`` has moved to
+        :mod:`werkzeug.middleware.proxy_fix`. This import will be
+        removed in 1.0.
+    """
+
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "'werkzeug.contrib.fixers.ProxyFix' has moved to 'werkzeug"
+            ".middleware.proxy_fix.ProxyFix'. This import is deprecated"
+            " as of version 0.15 and will be removed in 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super(ProxyFix, self).__init__(*args, **kwargs)
+
+
+class HeaderRewriterFix(object):
+    """This middleware can remove response headers and add others.  This
+    is for example useful to remove the `Date` header from responses if you
+    are using a server that adds that header, no matter if it's present or
+    not or to add `X-Powered-By` headers::
+
+        app = HeaderRewriterFix(app, remove_headers=['Date'],
+                                add_headers=[('X-Powered-By', 'WSGI')])
+
+    :param app: the WSGI application
+    :param remove_headers: a sequence of header keys that should be
+                           removed.
+    :param add_headers: a sequence of ``(key, value)`` tuples that should
+                        be added.
+
+    .. deprecated:: 0.15
+        This middleware will be removed in 1.0.
+    """
+
+    def __init__(self, app, remove_headers=None, add_headers=None):
+        warnings.warn(
+            "'HeaderRewriterFix' is deprecated as of version 0.15 and"
+            " will be removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.app = app
+        self.remove_headers = set(x.lower() for x in (remove_headers or ()))
+        self.add_headers = list(add_headers or ())
+
+    def __call__(self, environ, start_response):
+        def rewriting_start_response(status, headers, exc_info=None):
+            new_headers = []
+            for key, value in headers:
+                if key.lower() not in self.remove_headers:
+                    new_headers.append((key, value))
+            new_headers += self.add_headers
+            return start_response(status, new_headers, exc_info)
+
+        return self.app(environ, rewriting_start_response)
+
+
+class InternetExplorerFix(object):
+    """This middleware fixes a couple of bugs with Microsoft Internet
+    Explorer.  Currently the following fixes are applied:
+
+    -   removing of `Vary` headers for unsupported mimetypes which
+        causes troubles with caching.  Can be disabled by passing
+        ``fix_vary=False`` to the constructor.
+        see: https://support.microsoft.com/en-us/help/824847
+
+    -   removes offending headers to work around caching bugs in
+        Internet Explorer if `Content-Disposition` is set.  Can be
+        disabled by passing ``fix_attach=False`` to the constructor.
+
+    If it does not detect affected Internet Explorer versions it won't touch
+    the request / response.
+
+    .. deprecated:: 0.15
+        This middleware will be removed in 1.0.
+    """
+
+    # This code was inspired by Django fixers for the same bugs.  The
+    # fix_vary and fix_attach fixers were originally implemented in Django
+    # by Michael Axiak and is available as part of the Django project:
+    #     https://code.djangoproject.com/ticket/4148
+
+    def __init__(self, app, fix_vary=True, fix_attach=True):
+        warnings.warn(
+            "'InternetExplorerFix' is deprecated as of version 0.15 and"
+            " will be removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.app = app
+        self.fix_vary = fix_vary
+        self.fix_attach = fix_attach
+
+    def fix_headers(self, environ, headers, status=None):
+        if self.fix_vary:
+            header = headers.get("content-type", "")
+            mimetype, options = parse_options_header(header)
+            if mimetype not in ("text/html", "text/plain", "text/sgml"):
+                headers.pop("vary", None)
+
+        if self.fix_attach and "content-disposition" in headers:
+            pragma = parse_set_header(headers.get("pragma", ""))
+            pragma.discard("no-cache")
+            header = pragma.to_header()
+            if not header:
+                headers.pop("pragma", "")
+            else:
+                headers["Pragma"] = header
+            header = headers.get("cache-control", "")
+            if header:
+                cc = parse_cache_control_header(header, cls=ResponseCacheControl)
+                cc.no_cache = None
+                cc.no_store = False
+                header = cc.to_header()
+                if not header:
+                    headers.pop("cache-control", "")
+                else:
+                    headers["Cache-Control"] = header
+
+    def run_fixed(self, environ, start_response):
+        def fixing_start_response(status, headers, exc_info=None):
+            headers = Headers(headers)
+            self.fix_headers(environ, headers, status)
+            return start_response(status, headers.to_wsgi_list(), exc_info)
+
+        return self.app(environ, fixing_start_response)
+
+    def __call__(self, environ, start_response):
+        ua = UserAgent(environ)
+        if ua.browser != "msie":
+            return self.app(environ, start_response)
+        return self.run_fixed(environ, start_response)

python/werkzeug/contrib/iterio.py

@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+r"""
+    werkzeug.contrib.iterio
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    This module implements a :class:`IterIO` that converts an iterator into
+    a stream object and the other way round.  Converting streams into
+    iterators requires the `greenlet`_ module.
+
+    To convert an iterator into a stream all you have to do is to pass it
+    directly to the :class:`IterIO` constructor.  In this example we pass it
+    a newly created generator::
+
+        def foo():
+            yield "something\n"
+            yield "otherthings"
+        stream = IterIO(foo())
+        print stream.read()         # read the whole iterator
+
+    The other way round works a bit different because we have to ensure that
+    the code execution doesn't take place yet.  An :class:`IterIO` call with a
+    callable as first argument does two things.  The function itself is passed
+    an :class:`IterIO` stream it can feed.  The object returned by the
+    :class:`IterIO` constructor on the other hand is not an stream object but
+    an iterator::
+
+        def foo(stream):
+            stream.write("some")
+            stream.write("thing")
+            stream.flush()
+            stream.write("otherthing")
+        iterator = IterIO(foo)
+        print iterator.next()       # prints something
+        print iterator.next()       # prints otherthing
+        iterator.next()             # raises StopIteration
+
+    .. _greenlet: https://github.com/python-greenlet/greenlet
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import warnings
+
+from .._compat import implements_iterator
+
+try:
+    import greenlet
+except ImportError:
+    greenlet = None
+
+warnings.warn(
+    "'werkzeug.contrib.iterio' is deprecated as of version 0.15 and"
+    " will be removed in version 1.0.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
+def _mixed_join(iterable, sentinel):
+    """concatenate any string type in an intelligent way."""
+    iterator = iter(iterable)
+    first_item = next(iterator, sentinel)
+    if isinstance(first_item, bytes):
+        return first_item + b"".join(iterator)
+    return first_item + u"".join(iterator)
+
+
+def _newline(reference_string):
+    if isinstance(reference_string, bytes):
+        return b"\n"
+    return u"\n"
+
+
+@implements_iterator
+class IterIO(object):
+    """Instances of this object implement an interface compatible with the
+    standard Python :class:`file` object.  Streams are either read-only or
+    write-only depending on how the object is created.
+
+    If the first argument is an iterable a file like object is returned that
+    returns the contents of the iterable.  In case the iterable is empty
+    read operations will return the sentinel value.
+
+    If the first argument is a callable then the stream object will be
+    created and passed to that function.  The caller itself however will
+    not receive a stream but an iterable.  The function will be executed
+    step by step as something iterates over the returned iterable.  Each
+    call to :meth:`flush` will create an item for the iterable.  If
+    :meth:`flush` is called without any writes in-between the sentinel
+    value will be yielded.
+
+    Note for Python 3: due to the incompatible interface of bytes and
+    streams you should set the sentinel value explicitly to an empty
+    bytestring (``b''``) if you are expecting to deal with bytes as
+    otherwise the end of the stream is marked with the wrong sentinel
+    value.
+
+    .. versionadded:: 0.9
+       `sentinel` parameter was added.
+    """
+
+    def __new__(cls, obj, sentinel=""):
+        try:
+            iterator = iter(obj)
+        except TypeError:
+            return IterI(obj, sentinel)
+        return IterO(iterator, sentinel)
+
+    def __iter__(self):
+        return self
+
+    def tell(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        return self.pos
+
+    def isatty(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        return False
+
+    def seek(self, pos, mode=0):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def truncate(self, size=None):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def write(self, s):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def writelines(self, list):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def read(self, n=-1):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def readlines(self, sizehint=0):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def readline(self, length=None):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def flush(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        raise IOError(9, "Bad file descriptor")
+
+    def __next__(self):
+        if self.closed:
+            raise StopIteration()
+        line = self.readline()
+        if not line:
+            raise StopIteration()
+        return line
+
+
+class IterI(IterIO):
+    """Convert an stream into an iterator."""
+
+    def __new__(cls, func, sentinel=""):
+        if greenlet is None:
+            raise RuntimeError("IterI requires greenlet support")
+        stream = object.__new__(cls)
+        stream._parent = greenlet.getcurrent()
+        stream._buffer = []
+        stream.closed = False
+        stream.sentinel = sentinel
+        stream.pos = 0
+
+        def run():
+            func(stream)
+            stream.close()
+
+        g = greenlet.greenlet(run, stream._parent)
+        while 1:
+            rv = g.switch()
+            if not rv:
+                return
+            yield rv[0]
+
+    def close(self):
+        if not self.closed:
+            self.closed = True
+            self._flush_impl()
+
+    def write(self, s):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        if s:
+            self.pos += len(s)
+            self._buffer.append(s)
+
+    def writelines(self, list):
+        for item in list:
+            self.write(item)
+
+    def flush(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        self._flush_impl()
+
+    def _flush_impl(self):
+        data = _mixed_join(self._buffer, self.sentinel)
+        self._buffer = []
+        if not data and self.closed:
+            self._parent.switch()
+        else:
+            self._parent.switch((data,))
+
+
+class IterO(IterIO):
+    """Iter output.  Wrap an iterator and give it a stream like interface."""
+
+    def __new__(cls, gen, sentinel=""):
+        self = object.__new__(cls)
+        self._gen = gen
+        self._buf = None
+        self.sentinel = sentinel
+        self.closed = False
+        self.pos = 0
+        return self
+
+    def __iter__(self):
+        return self
+
+    def _buf_append(self, string):
+        """Replace string directly without appending to an empty string,
+        avoiding type issues."""
+        if not self._buf:
+            self._buf = string
+        else:
+            self._buf += string
+
+    def close(self):
+        if not self.closed:
+            self.closed = True
+            if hasattr(self._gen, "close"):
+                self._gen.close()
+
+    def seek(self, pos, mode=0):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        if mode == 1:
+            pos += self.pos
+        elif mode == 2:
+            self.read()
+            self.pos = min(self.pos, self.pos + pos)
+            return
+        elif mode != 0:
+            raise IOError("Invalid argument")
+        buf = []
+        try:
+            tmp_end_pos = len(self._buf or "")
+            while pos > tmp_end_pos:
+                item = next(self._gen)
+                tmp_end_pos += len(item)
+                buf.append(item)
+        except StopIteration:
+            pass
+        if buf:
+            self._buf_append(_mixed_join(buf, self.sentinel))
+        self.pos = max(0, pos)
+
+    def read(self, n=-1):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+        if n < 0:
+            self._buf_append(_mixed_join(self._gen, self.sentinel))
+            result = self._buf[self.pos :]
+            self.pos += len(result)
+            return result
+        new_pos = self.pos + n
+        buf = []
+        try:
+            tmp_end_pos = 0 if self._buf is None else len(self._buf)
+            while new_pos > tmp_end_pos or (self._buf is None and not buf):
+                item = next(self._gen)
+                tmp_end_pos += len(item)
+                buf.append(item)
+        except StopIteration:
+            pass
+        if buf:
+            self._buf_append(_mixed_join(buf, self.sentinel))
+
+        if self._buf is None:
+            return self.sentinel
+
+        new_pos = max(0, new_pos)
+        try:
+            return self._buf[self.pos : new_pos]
+        finally:
+            self.pos = min(new_pos, len(self._buf))
+
+    def readline(self, length=None):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+        nl_pos = -1
+        if self._buf:
+            nl_pos = self._buf.find(_newline(self._buf), self.pos)
+        buf = []
+        try:
+            if self._buf is None:
+                pos = self.pos
+            else:
+                pos = len(self._buf)
+            while nl_pos < 0:
+                item = next(self._gen)
+                local_pos = item.find(_newline(item))
+                buf.append(item)
+                if local_pos >= 0:
+                    nl_pos = pos + local_pos
+                    break
+                pos += len(item)
+        except StopIteration:
+            pass
+        if buf:
+            self._buf_append(_mixed_join(buf, self.sentinel))
+
+        if self._buf is None:
+            return self.sentinel
+
+        if nl_pos < 0:
+            new_pos = len(self._buf)
+        else:
+            new_pos = nl_pos + 1
+        if length is not None and self.pos + length < new_pos:
+            new_pos = self.pos + length
+        try:
+            return self._buf[self.pos : new_pos]
+        finally:
+            self.pos = min(new_pos, len(self._buf))
+
+    def readlines(self, sizehint=0):
+        total = 0
+        lines = []
+        line = self.readline()
+        while line:
+            lines.append(line)
+            total += len(line)
+            if 0 < sizehint <= total:
+                break
+            line = self.readline()
+        return lines

python/werkzeug/contrib/lint.py

@@ -0,0 +1,11 @@
+import warnings
+
+from ..middleware.lint import *  # noqa: F401, F403
+
+warnings.warn(
+    "'werkzeug.contrib.lint' has moved to 'werkzeug.middleware.lint'."
+    " This import is deprecated as of version 0.15 and will be removed"
+    " in version 1.0.",
+    DeprecationWarning,
+    stacklevel=2,
+)

python/werkzeug/contrib/profiler.py

@@ -0,0 +1,42 @@
+import warnings
+
+from ..middleware.profiler import *  # noqa: F401, F403
+
+warnings.warn(
+    "'werkzeug.contrib.profiler' has moved to"
+    "'werkzeug.middleware.profiler'. This import is deprecated as of"
+    "version 0.15 and will be removed in version 1.0.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
+class MergeStream(object):
+    """An object that redirects ``write`` calls to multiple streams.
+    Use this to log to both ``sys.stdout`` and a file::
+
+        f = open('profiler.log', 'w')
+        stream = MergeStream(sys.stdout, f)
+        profiler = ProfilerMiddleware(app, stream)
+
+    .. deprecated:: 0.15
+        Use the ``tee`` command in your terminal instead. This class
+        will be removed in 1.0.
+    """
+
+    def __init__(self, *streams):
+        warnings.warn(
+            "'MergeStream' is deprecated as of version 0.15 and will be removed in"
+            " version 1.0. Use your terminal's 'tee' command instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        if not streams:
+            raise TypeError("At least one stream must be given.")
+
+        self.streams = streams
+
+    def write(self, data):
+        for stream in self.streams:
+            stream.write(data)

python/werkzeug/contrib/securecookie.py

@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+r"""
+    werkzeug.contrib.securecookie
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This module implements a cookie that is not alterable from the client
+    because it adds a checksum the server checks for.  You can use it as
+    session replacement if all you have is a user id or something to mark
+    a logged in user.
+
+    Keep in mind that the data is still readable from the client as a
+    normal cookie is.  However you don't have to store and flush the
+    sessions you have at the server.
+
+    Example usage:
+
+    >>> from werkzeug.contrib.securecookie import SecureCookie
+    >>> x = SecureCookie({"foo": 42, "baz": (1, 2, 3)}, "deadbeef")
+
+    Dumping into a string so that one can store it in a cookie:
+
+    >>> value = x.serialize()
+
+    Loading from that string again:
+
+    >>> x = SecureCookie.unserialize(value, "deadbeef")
+    >>> x["baz"]
+    (1, 2, 3)
+
+    If someone modifies the cookie and the checksum is wrong the unserialize
+    method will fail silently and return a new empty `SecureCookie` object.
+
+    Keep in mind that the values will be visible in the cookie so do not
+    store data in a cookie you don't want the user to see.
+
+    Application Integration
+    =======================
+
+    If you are using the werkzeug request objects you could integrate the
+    secure cookie into your application like this::
+
+        from werkzeug.utils import cached_property
+        from werkzeug.wrappers import BaseRequest
+        from werkzeug.contrib.securecookie import SecureCookie
+
+        # don't use this key but a different one; you could just use
+        # os.urandom(20) to get something random
+        SECRET_KEY = '\xfa\xdd\xb8z\xae\xe0}4\x8b\xea'
+
+        class Request(BaseRequest):
+
+            @cached_property
+            def client_session(self):
+                data = self.cookies.get('session_data')
+                if not data:
+                    return SecureCookie(secret_key=SECRET_KEY)
+                return SecureCookie.unserialize(data, SECRET_KEY)
+
+        def application(environ, start_response):
+            request = Request(environ)
+
+            # get a response object here
+            response = ...
+
+            if request.client_session.should_save:
+                session_data = request.client_session.serialize()
+                response.set_cookie('session_data', session_data,
+                                    httponly=True)
+            return response(environ, start_response)
+
+    A less verbose integration can be achieved by using shorthand methods::
+
+        class Request(BaseRequest):
+
+            @cached_property
+            def client_session(self):
+                return SecureCookie.load_cookie(self, secret_key=COOKIE_SECRET)
+
+        def application(environ, start_response):
+            request = Request(environ)
+
+            # get a response object here
+            response = ...
+
+            request.client_session.save_cookie(response)
+            return response(environ, start_response)
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import base64
+import pickle
+import warnings
+from hashlib import sha1 as _default_hash
+from hmac import new as hmac
+from time import time
+
+from .._compat import iteritems
+from .._compat import text_type
+from .._compat import to_bytes
+from .._compat import to_native
+from .._internal import _date_to_unix
+from ..contrib.sessions import ModificationTrackingDict
+from ..security import safe_str_cmp
+from ..urls import url_quote_plus
+from ..urls import url_unquote_plus
+
+warnings.warn(
+    "'werkzeug.contrib.securecookie' is deprecated as of version 0.15"
+    " and will be removed in version 1.0. It has moved to"
+    " https://github.com/pallets/secure-cookie.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
+class UnquoteError(Exception):
+    """Internal exception used to signal failures on quoting."""
+
+
+class SecureCookie(ModificationTrackingDict):
+    """Represents a secure cookie.  You can subclass this class and provide
+    an alternative mac method.  The import thing is that the mac method
+    is a function with a similar interface to the hashlib.  Required
+    methods are update() and digest().
+
+    Example usage:
+
+    >>> x = SecureCookie({"foo": 42, "baz": (1, 2, 3)}, "deadbeef")
+    >>> x["foo"]
+    42
+    >>> x["baz"]
+    (1, 2, 3)
+    >>> x["blafasel"] = 23
+    >>> x.should_save
+    True
+
+    :param data: the initial data.  Either a dict, list of tuples or `None`.
+    :param secret_key: the secret key.  If not set `None` or not specified
+                       it has to be set before :meth:`serialize` is called.
+    :param new: The initial value of the `new` flag.
+    """
+
+    #: The hash method to use.  This has to be a module with a new function
+    #: or a function that creates a hashlib object.  Such as `hashlib.md5`
+    #: Subclasses can override this attribute.  The default hash is sha1.
+    #: Make sure to wrap this in staticmethod() if you store an arbitrary
+    #: function there such as hashlib.sha1 which  might be implemented
+    #: as a function.
+    hash_method = staticmethod(_default_hash)
+
+    #: The module used for serialization. Should have a ``dumps`` and a
+    #: ``loads`` method that takes bytes. The default is :mod:`pickle`.
+    #:
+    #: .. versionchanged:: 0.15
+    #:     The default of ``pickle`` will change to :mod:`json` in 1.0.
+    serialization_method = pickle
+
+    #: if the contents should be base64 quoted.  This can be disabled if the
+    #: serialization process returns cookie safe strings only.
+    quote_base64 = True
+
+    def __init__(self, data=None, secret_key=None, new=True):
+        ModificationTrackingDict.__init__(self, data or ())
+        # explicitly convert it into a bytestring because python 2.6
+        # no longer performs an implicit string conversion on hmac
+        if secret_key is not None:
+            secret_key = to_bytes(secret_key, "utf-8")
+        self.secret_key = secret_key
+        self.new = new
+
+        if self.serialization_method is pickle:
+            warnings.warn(
+                "The default 'SecureCookie.serialization_method' will"
+                " change from pickle to json in version 1.0. To upgrade"
+                " existing tokens, override 'unquote' to try pickle if"
+                " json fails.",
+                stacklevel=2,
+            )
+
+    def __repr__(self):
+        return "<%s %s%s>" % (
+            self.__class__.__name__,
+            dict.__repr__(self),
+            "*" if self.should_save else "",
+        )
+
+    @property
+    def should_save(self):
+        """True if the session should be saved.  By default this is only true
+        for :attr:`modified` cookies, not :attr:`new`.
+        """
+        return self.modified
+
+    @classmethod
+    def quote(cls, value):
+        """Quote the value for the cookie.  This can be any object supported
+        by :attr:`serialization_method`.
+
+        :param value: the value to quote.
+        """
+        if cls.serialization_method is not None:
+            value = cls.serialization_method.dumps(value)
+        if cls.quote_base64:
+            value = b"".join(
+                base64.b64encode(to_bytes(value, "utf8")).splitlines()
+            ).strip()
+        return value
+
+    @classmethod
+    def unquote(cls, value):
+        """Unquote the value for the cookie.  If unquoting does not work a
+        :exc:`UnquoteError` is raised.
+
+        :param value: the value to unquote.
+        """
+        try:
+            if cls.quote_base64:
+                value = base64.b64decode(value)
+            if cls.serialization_method is not None:
+                value = cls.serialization_method.loads(value)
+            return value
+        except Exception:
+            # unfortunately pickle and other serialization modules can
+            # cause pretty every error here.  if we get one we catch it
+            # and convert it into an UnquoteError
+            raise UnquoteError()
+
+    def serialize(self, expires=None):
+        """Serialize the secure cookie into a string.
+
+        If expires is provided, the session will be automatically invalidated
+        after expiration when you unseralize it. This provides better
+        protection against session cookie theft.
+
+        :param expires: an optional expiration date for the cookie (a
+                        :class:`datetime.datetime` object)
+        """
+        if self.secret_key is None:
+            raise RuntimeError("no secret key defined")
+        if expires:
+            self["_expires"] = _date_to_unix(expires)
+        result = []
+        mac = hmac(self.secret_key, None, self.hash_method)
+        for key, value in sorted(self.items()):
+            result.append(
+                (
+                    "%s=%s" % (url_quote_plus(key), self.quote(value).decode("ascii"))
+                ).encode("ascii")
+            )
+            mac.update(b"|" + result[-1])
+        return b"?".join([base64.b64encode(mac.digest()).strip(), b"&".join(result)])
+
+    @classmethod
+    def unserialize(cls, string, secret_key):
+        """Load the secure cookie from a serialized string.
+
+        :param string: the cookie value to unserialize.
+        :param secret_key: the secret key used to serialize the cookie.
+        :return: a new :class:`SecureCookie`.
+        """
+        if isinstance(string, text_type):
+            string = string.encode("utf-8", "replace")
+        if isinstance(secret_key, text_type):
+            secret_key = secret_key.encode("utf-8", "replace")
+        try:
+            base64_hash, data = string.split(b"?", 1)
+        except (ValueError, IndexError):
+            items = ()
+        else:
+            items = {}
+            mac = hmac(secret_key, None, cls.hash_method)
+            for item in data.split(b"&"):
+                mac.update(b"|" + item)
+                if b"=" not in item:
+                    items = None
+                    break
+                key, value = item.split(b"=", 1)
+                # try to make the key a string
+                key = url_unquote_plus(key.decode("ascii"))
+                try:
+                    key = to_native(key)
+                except UnicodeError:
+                    pass
+                items[key] = value
+
+            # no parsing error and the mac looks okay, we can now
+            # sercurely unpickle our cookie.
+            try:
+                client_hash = base64.b64decode(base64_hash)
+            except TypeError:
+                items = client_hash = None
+            if items is not None and safe_str_cmp(client_hash, mac.digest()):
+                try:
+                    for key, value in iteritems(items):
+                        items[key] = cls.unquote(value)
+                except UnquoteError:
+                    items = ()
+                else:
+                    if "_expires" in items:
+                        if time() > items["_expires"]:
+                            items = ()
+                        else:
+                            del items["_expires"]
+            else:
+                items = ()
+        return cls(items, secret_key, False)
+
+    @classmethod
+    def load_cookie(cls, request, key="session", secret_key=None):
+        """Loads a :class:`SecureCookie` from a cookie in request.  If the
+        cookie is not set, a new :class:`SecureCookie` instanced is
+        returned.
+
+        :param request: a request object that has a `cookies` attribute
+                        which is a dict of all cookie values.
+        :param key: the name of the cookie.
+        :param secret_key: the secret key used to unquote the cookie.
+                           Always provide the value even though it has
+                           no default!
+        """
+        data = request.cookies.get(key)
+        if not data:
+            return cls(secret_key=secret_key)
+        return cls.unserialize(data, secret_key)
+
+    def save_cookie(
+        self,
+        response,
+        key="session",
+        expires=None,
+        session_expires=None,
+        max_age=None,
+        path="/",
+        domain=None,
+        secure=None,
+        httponly=False,
+        force=False,
+    ):
+        """Saves the SecureCookie in a cookie on response object.  All
+        parameters that are not described here are forwarded directly
+        to :meth:`~BaseResponse.set_cookie`.
+
+        :param response: a response object that has a
+                         :meth:`~BaseResponse.set_cookie` method.
+        :param key: the name of the cookie.
+        :param session_expires: the expiration date of the secure cookie
+                                stored information.  If this is not provided
+                                the cookie `expires` date is used instead.
+        """
+        if force or self.should_save:
+            data = self.serialize(session_expires or expires)
+            response.set_cookie(
+                key,
+                data,
+                expires=expires,
+                max_age=max_age,
+                path=path,
+                domain=domain,
+                secure=secure,
+                httponly=httponly,
+            )

python/werkzeug/contrib/sessions.py

@@ -0,0 +1,389 @@
+# -*- coding: utf-8 -*-
+r"""
+    werkzeug.contrib.sessions
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This module contains some helper classes that help one to add session
+    support to a python WSGI application.  For full client-side session
+    storage see :mod:`~werkzeug.contrib.securecookie` which implements a
+    secure, client-side session storage.
+
+
+    Application Integration
+    =======================
+
+    ::
+
+        from werkzeug.contrib.sessions import SessionMiddleware, \
+             FilesystemSessionStore
+
+        app = SessionMiddleware(app, FilesystemSessionStore())
+
+    The current session will then appear in the WSGI environment as
+    `werkzeug.session`.  However it's recommended to not use the middleware
+    but the stores directly in the application.  However for very simple
+    scripts a middleware for sessions could be sufficient.
+
+    This module does not implement methods or ways to check if a session is
+    expired.  That should be done by a cronjob and storage specific.  For
+    example to prune unused filesystem sessions one could check the modified
+    time of the files.  If sessions are stored in the database the new()
+    method should add an expiration timestamp for the session.
+
+    For better flexibility it's recommended to not use the middleware but the
+    store and session object directly in the application dispatching::
+
+        session_store = FilesystemSessionStore()
+
+        def application(environ, start_response):
+            request = Request(environ)
+            sid = request.cookies.get('cookie_name')
+            if sid is None:
+                request.session = session_store.new()
+            else:
+                request.session = session_store.get(sid)
+            response = get_the_response_object(request)
+            if request.session.should_save:
+                session_store.save(request.session)
+                response.set_cookie('cookie_name', request.session.sid)
+            return response(environ, start_response)
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import os
+import re
+import tempfile
+import warnings
+from hashlib import sha1
+from os import path
+from pickle import dump
+from pickle import HIGHEST_PROTOCOL
+from pickle import load
+from random import random
+from time import time
+
+from .._compat import PY2
+from .._compat import text_type
+from ..datastructures import CallbackDict
+from ..filesystem import get_filesystem_encoding
+from ..posixemulation import rename
+from ..utils import dump_cookie
+from ..utils import parse_cookie
+from ..wsgi import ClosingIterator
+
+warnings.warn(
+    "'werkzeug.contrib.sessions' is deprecated as of version 0.15 and"
+    " will be removed in version 1.0. It has moved to"
+    " https://github.com/pallets/secure-cookie.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+_sha1_re = re.compile(r"^[a-f0-9]{40}$")
+
+
+def _urandom():
+    if hasattr(os, "urandom"):
+        return os.urandom(30)
+    return text_type(random()).encode("ascii")
+
+
+def generate_key(salt=None):
+    if salt is None:
+        salt = repr(salt).encode("ascii")
+    return sha1(b"".join([salt, str(time()).encode("ascii"), _urandom()])).hexdigest()
+
+
+class ModificationTrackingDict(CallbackDict):
+    __slots__ = ("modified",)
+
+    def __init__(self, *args, **kwargs):
+        def on_update(self):
+            self.modified = True
+
+        self.modified = False
+        CallbackDict.__init__(self, on_update=on_update)
+        dict.update(self, *args, **kwargs)
+
+    def copy(self):
+        """Create a flat copy of the dict."""
+        missing = object()
+        result = object.__new__(self.__class__)
+        for name in self.__slots__:
+            val = getattr(self, name, missing)
+            if val is not missing:
+                setattr(result, name, val)
+        return result
+
+    def __copy__(self):
+        return self.copy()
+
+
+class Session(ModificationTrackingDict):
+    """Subclass of a dict that keeps track of direct object changes.  Changes
+    in mutable structures are not tracked, for those you have to set
+    `modified` to `True` by hand.
+    """
+
+    __slots__ = ModificationTrackingDict.__slots__ + ("sid", "new")
+
+    def __init__(self, data, sid, new=False):
+        ModificationTrackingDict.__init__(self, data)
+        self.sid = sid
+        self.new = new
+
+    def __repr__(self):
+        return "<%s %s%s>" % (
+            self.__class__.__name__,
+            dict.__repr__(self),
+            "*" if self.should_save else "",
+        )
+
+    @property
+    def should_save(self):
+        """True if the session should be saved.
+
+        .. versionchanged:: 0.6
+           By default the session is now only saved if the session is
+           modified, not if it is new like it was before.
+        """
+        return self.modified
+
+
+class SessionStore(object):
+    """Baseclass for all session stores.  The Werkzeug contrib module does not
+    implement any useful stores besides the filesystem store, application
+    developers are encouraged to create their own stores.
+
+    :param session_class: The session class to use.  Defaults to
+                          :class:`Session`.
+    """
+
+    def __init__(self, session_class=None):
+        if session_class is None:
+            session_class = Session
+        self.session_class = session_class
+
+    def is_valid_key(self, key):
+        """Check if a key has the correct format."""
+        return _sha1_re.match(key) is not None
+
+    def generate_key(self, salt=None):
+        """Simple function that generates a new session key."""
+        return generate_key(salt)
+
+    def new(self):
+        """Generate a new session."""
+        return self.session_class({}, self.generate_key(), True)
+
+    def save(self, session):
+        """Save a session."""
+
+    def save_if_modified(self, session):
+        """Save if a session class wants an update."""
+        if session.should_save:
+            self.save(session)
+
+    def delete(self, session):
+        """Delete a session."""
+
+    def get(self, sid):
+        """Get a session for this sid or a new session object.  This method
+        has to check if the session key is valid and create a new session if
+        that wasn't the case.
+        """
+        return self.session_class({}, sid, True)
+
+
+#: used for temporary files by the filesystem session store
+_fs_transaction_suffix = ".__wz_sess"
+
+
+class FilesystemSessionStore(SessionStore):
+    """Simple example session store that saves sessions on the filesystem.
+    This store works best on POSIX systems and Windows Vista / Windows
+    Server 2008 and newer.
+
+    .. versionchanged:: 0.6
+       `renew_missing` was added.  Previously this was considered `True`,
+       now the default changed to `False` and it can be explicitly
+       deactivated.
+
+    :param path: the path to the folder used for storing the sessions.
+                 If not provided the default temporary directory is used.
+    :param filename_template: a string template used to give the session
+                              a filename.  ``%s`` is replaced with the
+                              session id.
+    :param session_class: The session class to use.  Defaults to
+                          :class:`Session`.
+    :param renew_missing: set to `True` if you want the store to
+                          give the user a new sid if the session was
+                          not yet saved.
+    """
+
+    def __init__(
+        self,
+        path=None,
+        filename_template="werkzeug_%s.sess",
+        session_class=None,
+        renew_missing=False,
+        mode=0o644,
+    ):
+        SessionStore.__init__(self, session_class)
+        if path is None:
+            path = tempfile.gettempdir()
+        self.path = path
+        if isinstance(filename_template, text_type) and PY2:
+            filename_template = filename_template.encode(get_filesystem_encoding())
+        assert not filename_template.endswith(_fs_transaction_suffix), (
+            "filename templates may not end with %s" % _fs_transaction_suffix
+        )
+        self.filename_template = filename_template
+        self.renew_missing = renew_missing
+        self.mode = mode
+
+    def get_session_filename(self, sid):
+        # out of the box, this should be a strict ASCII subset but
+        # you might reconfigure the session object to have a more
+        # arbitrary string.
+        if isinstance(sid, text_type) and PY2:
+            sid = sid.encode(get_filesystem_encoding())
+        return path.join(self.path, self.filename_template % sid)
+
+    def save(self, session):
+        fn = self.get_session_filename(session.sid)
+        fd, tmp = tempfile.mkstemp(suffix=_fs_transaction_suffix, dir=self.path)
+        f = os.fdopen(fd, "wb")
+        try:
+            dump(dict(session), f, HIGHEST_PROTOCOL)
+        finally:
+            f.close()
+        try:
+            rename(tmp, fn)
+            os.chmod(fn, self.mode)
+        except (IOError, OSError):
+            pass
+
+    def delete(self, session):
+        fn = self.get_session_filename(session.sid)
+        try:
+            os.unlink(fn)
+        except OSError:
+            pass
+
+    def get(self, sid):
+        if not self.is_valid_key(sid):
+            return self.new()
+        try:
+            f = open(self.get_session_filename(sid), "rb")
+        except IOError:
+            if self.renew_missing:
+                return self.new()
+            data = {}
+        else:
+            try:
+                try:
+                    data = load(f)
+                except Exception:
+                    data = {}
+            finally:
+                f.close()
+        return self.session_class(data, sid, False)
+
+    def list(self):
+        """Lists all sessions in the store.
+
+        .. versionadded:: 0.6
+        """
+        before, after = self.filename_template.split("%s", 1)
+        filename_re = re.compile(
+            r"%s(.{5,})%s$" % (re.escape(before), re.escape(after))
+        )
+        result = []
+        for filename in os.listdir(self.path):
+            #: this is a session that is still being saved.
+            if filename.endswith(_fs_transaction_suffix):
+                continue
+            match = filename_re.match(filename)
+            if match is not None:
+                result.append(match.group(1))
+        return result
+
+
+class SessionMiddleware(object):
+    """A simple middleware that puts the session object of a store provided
+    into the WSGI environ.  It automatically sets cookies and restores
+    sessions.
+
+    However a middleware is not the preferred solution because it won't be as
+    fast as sessions managed by the application itself and will put a key into
+    the WSGI environment only relevant for the application which is against
+    the concept of WSGI.
+
+    The cookie parameters are the same as for the :func:`~dump_cookie`
+    function just prefixed with ``cookie_``.  Additionally `max_age` is
+    called `cookie_age` and not `cookie_max_age` because of backwards
+    compatibility.
+    """
+
+    def __init__(
+        self,
+        app,
+        store,
+        cookie_name="session_id",
+        cookie_age=None,
+        cookie_expires=None,
+        cookie_path="/",
+        cookie_domain=None,
+        cookie_secure=None,
+        cookie_httponly=False,
+        cookie_samesite="Lax",
+        environ_key="werkzeug.session",
+    ):
+        self.app = app
+        self.store = store
+        self.cookie_name = cookie_name
+        self.cookie_age = cookie_age
+        self.cookie_expires = cookie_expires
+        self.cookie_path = cookie_path
+        self.cookie_domain = cookie_domain
+        self.cookie_secure = cookie_secure
+        self.cookie_httponly = cookie_httponly
+        self.cookie_samesite = cookie_samesite
+        self.environ_key = environ_key
+
+    def __call__(self, environ, start_response):
+        cookie = parse_cookie(environ.get("HTTP_COOKIE", ""))
+        sid = cookie.get(self.cookie_name, None)
+        if sid is None:
+            session = self.store.new()
+        else:
+            session = self.store.get(sid)
+        environ[self.environ_key] = session
+
+        def injecting_start_response(status, headers, exc_info=None):
+            if session.should_save:
+                self.store.save(session)
+                headers.append(
+                    (
+                        "Set-Cookie",
+                        dump_cookie(
+                            self.cookie_name,
+                            session.sid,
+                            self.cookie_age,
+                            self.cookie_expires,
+                            self.cookie_path,
+                            self.cookie_domain,
+                            self.cookie_secure,
+                            self.cookie_httponly,
+                            samesite=self.cookie_samesite,
+                        ),
+                    )
+                )
+            return start_response(status, headers, exc_info)
+
+        return ClosingIterator(
+            self.app(environ, injecting_start_response),
+            lambda: self.store.save_if_modified(session),
+        )

python/werkzeug/contrib/wrappers.py

@@ -0,0 +1,385 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.contrib.wrappers
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Extra wrappers or mixins contributed by the community.  These wrappers can
+    be mixed in into request objects to add extra functionality.
+
+    Example::
+
+        from werkzeug.wrappers import Request as RequestBase
+        from werkzeug.contrib.wrappers import JSONRequestMixin
+
+        class Request(RequestBase, JSONRequestMixin):
+            pass
+
+    Afterwards this request object provides the extra functionality of the
+    :class:`JSONRequestMixin`.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import codecs
+import warnings
+
+from .._compat import wsgi_decoding_dance
+from ..exceptions import BadRequest
+from ..http import dump_options_header
+from ..http import parse_options_header
+from ..utils import cached_property
+from ..wrappers.json import JSONMixin as _JSONMixin
+
+
+def is_known_charset(charset):
+    """Checks if the given charset is known to Python."""
+    try:
+        codecs.lookup(charset)
+    except LookupError:
+        return False
+    return True
+
+
+class JSONRequestMixin(_JSONMixin):
+    """
+    .. deprecated:: 0.15
+        Moved to :class:`werkzeug.wrappers.json.JSONMixin`. This old
+        import will be removed in version 1.0.
+    """
+
+    @property
+    def json(self):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.JSONRequestMixin' has moved to"
+            " 'werkzeug.wrappers.json.JSONMixin'. This old import will"
+            " be removed in version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return super(JSONRequestMixin, self).json
+
+
+class ProtobufRequestMixin(object):
+
+    """Add protobuf parsing method to a request object.  This will parse the
+    input data through `protobuf`_ if possible.
+
+    :exc:`~werkzeug.exceptions.BadRequest` will be raised if the content-type
+    is not protobuf or if the data itself cannot be parsed property.
+
+    .. _protobuf: https://github.com/protocolbuffers/protobuf
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
+    """
+
+    #: by default the :class:`ProtobufRequestMixin` will raise a
+    #: :exc:`~werkzeug.exceptions.BadRequest` if the object is not
+    #: initialized.  You can bypass that check by setting this
+    #: attribute to `False`.
+    protobuf_check_initialization = True
+
+    def parse_protobuf(self, proto_type):
+        """Parse the data into an instance of proto_type."""
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.ProtobufRequestMixin' is"
+            " deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if "protobuf" not in self.environ.get("CONTENT_TYPE", ""):
+            raise BadRequest("Not a Protobuf request")
+
+        obj = proto_type()
+        try:
+            obj.ParseFromString(self.data)
+        except Exception:
+            raise BadRequest("Unable to parse Protobuf request")
+
+        # Fail if not all required fields are set
+        if self.protobuf_check_initialization and not obj.IsInitialized():
+            raise BadRequest("Partial Protobuf request")
+
+        return obj
+
+
+class RoutingArgsRequestMixin(object):
+
+    """This request mixin adds support for the wsgiorg routing args
+    `specification`_.
+
+    .. _specification: https://wsgi.readthedocs.io/en/latest/
+       specifications/routing_args.html
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
+    """
+
+    def _get_routing_args(self):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.RoutingArgsRequestMixin' is"
+            " deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.environ.get("wsgiorg.routing_args", (()))[0]
+
+    def _set_routing_args(self, value):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.RoutingArgsRequestMixin' is"
+            " deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if self.shallow:
+            raise RuntimeError(
+                "A shallow request tried to modify the WSGI "
+                "environment.  If you really want to do that, "
+                "set `shallow` to False."
+            )
+        self.environ["wsgiorg.routing_args"] = (value, self.routing_vars)
+
+    routing_args = property(
+        _get_routing_args,
+        _set_routing_args,
+        doc="""
+        The positional URL arguments as `tuple`.""",
+    )
+    del _get_routing_args, _set_routing_args
+
+    def _get_routing_vars(self):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.RoutingArgsRequestMixin' is"
+            " deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        rv = self.environ.get("wsgiorg.routing_args")
+        if rv is not None:
+            return rv[1]
+        rv = {}
+        if not self.shallow:
+            self.routing_vars = rv
+        return rv
+
+    def _set_routing_vars(self, value):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.RoutingArgsRequestMixin' is"
+            " deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if self.shallow:
+            raise RuntimeError(
+                "A shallow request tried to modify the WSGI "
+                "environment.  If you really want to do that, "
+                "set `shallow` to False."
+            )
+        self.environ["wsgiorg.routing_args"] = (self.routing_args, value)
+
+    routing_vars = property(
+        _get_routing_vars,
+        _set_routing_vars,
+        doc="""
+        The keyword URL arguments as `dict`.""",
+    )
+    del _get_routing_vars, _set_routing_vars
+
+
+class ReverseSlashBehaviorRequestMixin(object):
+
+    """This mixin reverses the trailing slash behavior of :attr:`script_root`
+    and :attr:`path`.  This makes it possible to use :func:`~urlparse.urljoin`
+    directly on the paths.
+
+    Because it changes the behavior or :class:`Request` this class has to be
+    mixed in *before* the actual request class::
+
+        class MyRequest(ReverseSlashBehaviorRequestMixin, Request):
+            pass
+
+    This example shows the differences (for an application mounted on
+    `/application` and the request going to `/application/foo/bar`):
+
+        +---------------+-------------------+---------------------+
+        |               | normal behavior   | reverse behavior    |
+        +===============+===================+=====================+
+        | `script_root` | ``/application``  | ``/application/``   |
+        +---------------+-------------------+---------------------+
+        | `path`        | ``/foo/bar``      | ``foo/bar``         |
+        +---------------+-------------------+---------------------+
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
+    """
+
+    @cached_property
+    def path(self):
+        """Requested path as unicode.  This works a bit like the regular path
+        info in the WSGI environment but will not include a leading slash.
+        """
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.ReverseSlashBehaviorRequestMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        path = wsgi_decoding_dance(
+            self.environ.get("PATH_INFO") or "", self.charset, self.encoding_errors
+        )
+        return path.lstrip("/")
+
+    @cached_property
+    def script_root(self):
+        """The root path of the script includling a trailing slash."""
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.ReverseSlashBehaviorRequestMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        path = wsgi_decoding_dance(
+            self.environ.get("SCRIPT_NAME") or "", self.charset, self.encoding_errors
+        )
+        return path.rstrip("/") + "/"
+
+
+class DynamicCharsetRequestMixin(object):
+
+    """"If this mixin is mixed into a request class it will provide
+    a dynamic `charset` attribute.  This means that if the charset is
+    transmitted in the content type headers it's used from there.
+
+    Because it changes the behavior or :class:`Request` this class has
+    to be mixed in *before* the actual request class::
+
+        class MyRequest(DynamicCharsetRequestMixin, Request):
+            pass
+
+    By default the request object assumes that the URL charset is the
+    same as the data charset.  If the charset varies on each request
+    based on the transmitted data it's not a good idea to let the URLs
+    change based on that.  Most browsers assume either utf-8 or latin1
+    for the URLs if they have troubles figuring out.  It's strongly
+    recommended to set the URL charset to utf-8::
+
+        class MyRequest(DynamicCharsetRequestMixin, Request):
+            url_charset = 'utf-8'
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
+
+    .. versionadded:: 0.6
+    """
+
+    #: the default charset that is assumed if the content type header
+    #: is missing or does not contain a charset parameter.  The default
+    #: is latin1 which is what HTTP specifies as default charset.
+    #: You may however want to set this to utf-8 to better support
+    #: browsers that do not transmit a charset for incoming data.
+    default_charset = "latin1"
+
+    def unknown_charset(self, charset):
+        """Called if a charset was provided but is not supported by
+        the Python codecs module.  By default latin1 is assumed then
+        to not lose any information, you may override this method to
+        change the behavior.
+
+        :param charset: the charset that was not found.
+        :return: the replacement charset.
+        """
+        return "latin1"
+
+    @cached_property
+    def charset(self):
+        """The charset from the content type."""
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.DynamicCharsetRequestMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        header = self.environ.get("CONTENT_TYPE")
+        if header:
+            ct, options = parse_options_header(header)
+            charset = options.get("charset")
+            if charset:
+                if is_known_charset(charset):
+                    return charset
+                return self.unknown_charset(charset)
+        return self.default_charset
+
+
+class DynamicCharsetResponseMixin(object):
+
+    """If this mixin is mixed into a response class it will provide
+    a dynamic `charset` attribute.  This means that if the charset is
+    looked up and stored in the `Content-Type` header and updates
+    itself automatically.  This also means a small performance hit but
+    can be useful if you're working with different charsets on
+    responses.
+
+    Because the charset attribute is no a property at class-level, the
+    default value is stored in `default_charset`.
+
+    Because it changes the behavior or :class:`Response` this class has
+    to be mixed in *before* the actual response class::
+
+        class MyResponse(DynamicCharsetResponseMixin, Response):
+            pass
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
+
+    .. versionadded:: 0.6
+    """
+
+    #: the default charset.
+    default_charset = "utf-8"
+
+    def _get_charset(self):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.DynamicCharsetResponseMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        header = self.headers.get("content-type")
+        if header:
+            charset = parse_options_header(header)[1].get("charset")
+            if charset:
+                return charset
+        return self.default_charset
+
+    def _set_charset(self, charset):
+        warnings.warn(
+            "'werkzeug.contrib.wrappers.DynamicCharsetResponseMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        header = self.headers.get("content-type")
+        ct, options = parse_options_header(header)
+        if not ct:
+            raise TypeError("Cannot set charset if Content-Type header is missing.")
+        options["charset"] = charset
+        self.headers["Content-Type"] = dump_options_header(ct, options)
+
+    charset = property(
+        _get_charset,
+        _set_charset,
+        doc="""
+        The charset for the response.  It's stored inside the
+        Content-Type header as a parameter.""",
+    )
+    del _get_charset, _set_charset

python/werkzeug/exceptions.py

@@ -0,0 +1,774 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.exceptions
+    ~~~~~~~~~~~~~~~~~~~
+
+    This module implements a number of Python exceptions you can raise from
+    within your views to trigger a standard non-200 response.
+
+
+    Usage Example
+    -------------
+
+    ::
+
+        from werkzeug.wrappers import BaseRequest
+        from werkzeug.wsgi import responder
+        from werkzeug.exceptions import HTTPException, NotFound
+
+        def view(request):
+            raise NotFound()
+
+        @responder
+        def application(environ, start_response):
+            request = BaseRequest(environ)
+            try:
+                return view(request)
+            except HTTPException as e:
+                return e
+
+
+    As you can see from this example those exceptions are callable WSGI
+    applications.  Because of Python 2.4 compatibility those do not extend
+    from the response objects but only from the python exception class.
+
+    As a matter of fact they are not Werkzeug response objects.  However you
+    can get a response object by calling ``get_response()`` on a HTTP
+    exception.
+
+    Keep in mind that you have to pass an environment to ``get_response()``
+    because some errors fetch additional information from the WSGI
+    environment.
+
+    If you want to hook in a different exception page to say, a 404 status
+    code, you can add a second except for a specific subclass of an error::
+
+        @responder
+        def application(environ, start_response):
+            request = BaseRequest(environ)
+            try:
+                return view(request)
+            except NotFound, e:
+                return not_found(request)
+            except HTTPException, e:
+                return e
+
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import sys
+
+import werkzeug
+
+# Because of bootstrapping reasons we need to manually patch ourselves
+# onto our parent module.
+werkzeug.exceptions = sys.modules[__name__]
+
+from ._compat import implements_to_string
+from ._compat import integer_types
+from ._compat import iteritems
+from ._compat import text_type
+from ._internal import _get_environ
+from .wrappers import Response
+
+
+@implements_to_string
+class HTTPException(Exception):
+    """Baseclass for all HTTP exceptions.  This exception can be called as WSGI
+    application to render a default error page or you can catch the subclasses
+    of it independently and render nicer error messages.
+    """
+
+    code = None
+    description = None
+
+    def __init__(self, description=None, response=None):
+        super(Exception, self).__init__()
+        if description is not None:
+            self.description = description
+        self.response = response
+
+    @classmethod
+    def wrap(cls, exception, name=None):
+        """Create an exception that is a subclass of the calling HTTP
+        exception and the ``exception`` argument.
+
+        The first argument to the class will be passed to the
+        wrapped ``exception``, the rest to the HTTP exception. If
+        ``self.args`` is not empty, the wrapped exception message is
+        added to the HTTP exception description.
+
+        .. versionchanged:: 0.15
+            The description includes the wrapped exception message.
+        """
+
+        class newcls(cls, exception):
+            def __init__(self, arg=None, *args, **kwargs):
+                super(cls, self).__init__(*args, **kwargs)
+
+                if arg is None:
+                    exception.__init__(self)
+                else:
+                    exception.__init__(self, arg)
+
+            def get_description(self, environ=None):
+                out = super(cls, self).get_description(environ=environ)
+
+                if self.args:
+                    out += "<p><pre><code>{}: {}</code></pre></p>".format(
+                        exception.__name__, escape(exception.__str__(self))
+                    )
+
+                return out
+
+        newcls.__module__ = sys._getframe(1).f_globals.get("__name__")
+        newcls.__name__ = name or cls.__name__ + exception.__name__
+        return newcls
+
+    @property
+    def name(self):
+        """The status name."""
+        return HTTP_STATUS_CODES.get(self.code, "Unknown Error")
+
+    def get_description(self, environ=None):
+        """Get the description."""
+        return u"<p>%s</p>" % escape(self.description)
+
+    def get_body(self, environ=None):
+        """Get the HTML body."""
+        return text_type(
+            (
+                u'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n'
+                u"<title>%(code)s %(name)s</title>\n"
+                u"<h1>%(name)s</h1>\n"
+                u"%(description)s\n"
+            )
+            % {
+                "code": self.code,
+                "name": escape(self.name),
+                "description": self.get_description(environ),
+            }
+        )
+
+    def get_headers(self, environ=None):
+        """Get a list of headers."""
+        return [("Content-Type", "text/html")]
+
+    def get_response(self, environ=None):
+        """Get a response object.  If one was passed to the exception
+        it's returned directly.
+
+        :param environ: the optional environ for the request.  This
+                        can be used to modify the response depending
+                        on how the request looked like.
+        :return: a :class:`Response` object or a subclass thereof.
+        """
+        if self.response is not None:
+            return self.response
+        if environ is not None:
+            environ = _get_environ(environ)
+        headers = self.get_headers(environ)
+        return Response(self.get_body(environ), self.code, headers)
+
+    def __call__(self, environ, start_response):
+        """Call the exception as WSGI application.
+
+        :param environ: the WSGI environment.
+        :param start_response: the response callable provided by the WSGI
+                               server.
+        """
+        response = self.get_response(environ)
+        return response(environ, start_response)
+
+    def __str__(self):
+        code = self.code if self.code is not None else "???"
+        return "%s %s: %s" % (code, self.name, self.description)
+
+    def __repr__(self):
+        code = self.code if self.code is not None else "???"
+        return "<%s '%s: %s'>" % (self.__class__.__name__, code, self.name)
+
+
+class BadRequest(HTTPException):
+    """*400* `Bad Request`
+
+    Raise if the browser sends something to the application the application
+    or server cannot handle.
+    """
+
+    code = 400
+    description = (
+        "The browser (or proxy) sent a request that this server could "
+        "not understand."
+    )
+
+
+class ClientDisconnected(BadRequest):
+    """Internal exception that is raised if Werkzeug detects a disconnected
+    client.  Since the client is already gone at that point attempting to
+    send the error message to the client might not work and might ultimately
+    result in another exception in the server.  Mainly this is here so that
+    it is silenced by default as far as Werkzeug is concerned.
+
+    Since disconnections cannot be reliably detected and are unspecified
+    by WSGI to a large extent this might or might not be raised if a client
+    is gone.
+
+    .. versionadded:: 0.8
+    """
+
+
+class SecurityError(BadRequest):
+    """Raised if something triggers a security error.  This is otherwise
+    exactly like a bad request error.
+
+    .. versionadded:: 0.9
+    """
+
+
+class BadHost(BadRequest):
+    """Raised if the submitted host is badly formatted.
+
+    .. versionadded:: 0.11.2
+    """
+
+
+class Unauthorized(HTTPException):
+    """*401* ``Unauthorized``
+
+    Raise if the user is not authorized to access a resource.
+
+    The ``www_authenticate`` argument should be used to set the
+    ``WWW-Authenticate`` header. This is used for HTTP basic auth and
+    other schemes. Use :class:`~werkzeug.datastructures.WWWAuthenticate`
+    to create correctly formatted values. Strictly speaking a 401
+    response is invalid if it doesn't provide at least one value for
+    this header, although real clients typically don't care.
+
+    :param description: Override the default message used for the body
+        of the response.
+    :param www-authenticate: A single value, or list of values, for the
+        WWW-Authenticate header.
+
+    .. versionchanged:: 0.15.3
+        If the ``www_authenticate`` argument is not set, the
+        ``WWW-Authenticate`` header is not set.
+
+    .. versionchanged:: 0.15.3
+        The ``response`` argument was restored.
+
+    .. versionchanged:: 0.15.1
+        ``description`` was moved back as the first argument, restoring
+         its previous position.
+
+    .. versionchanged:: 0.15.0
+        ``www_authenticate`` was added as the first argument, ahead of
+        ``description``.
+    """
+
+    code = 401
+    description = (
+        "The server could not verify that you are authorized to access"
+        " the URL requested. You either supplied the wrong credentials"
+        " (e.g. a bad password), or your browser doesn't understand"
+        " how to supply the credentials required."
+    )
+
+    def __init__(self, description=None, response=None, www_authenticate=None):
+        HTTPException.__init__(self, description, response)
+
+        if www_authenticate is not None:
+            if not isinstance(www_authenticate, (tuple, list)):
+                www_authenticate = (www_authenticate,)
+
+        self.www_authenticate = www_authenticate
+
+    def get_headers(self, environ=None):
+        headers = HTTPException.get_headers(self, environ)
+        if self.www_authenticate:
+            headers.append(
+                ("WWW-Authenticate", ", ".join([str(x) for x in self.www_authenticate]))
+            )
+        return headers
+
+
+class Forbidden(HTTPException):
+    """*403* `Forbidden`
+
+    Raise if the user doesn't have the permission for the requested resource
+    but was authenticated.
+    """
+
+    code = 403
+    description = (
+        "You don't have the permission to access the requested"
+        " resource. It is either read-protected or not readable by the"
+        " server."
+    )
+
+
+class NotFound(HTTPException):
+    """*404* `Not Found`
+
+    Raise if a resource does not exist and never existed.
+    """
+
+    code = 404
+    description = (
+        "The requested URL was not found on the server. If you entered"
+        " the URL manually please check your spelling and try again."
+    )
+
+
+class MethodNotAllowed(HTTPException):
+    """*405* `Method Not Allowed`
+
+    Raise if the server used a method the resource does not handle.  For
+    example `POST` if the resource is view only.  Especially useful for REST.
+
+    The first argument for this exception should be a list of allowed methods.
+    Strictly speaking the response would be invalid if you don't provide valid
+    methods in the header which you can do with that list.
+    """
+
+    code = 405
+    description = "The method is not allowed for the requested URL."
+
+    def __init__(self, valid_methods=None, description=None):
+        """Takes an optional list of valid http methods
+        starting with werkzeug 0.3 the list will be mandatory."""
+        HTTPException.__init__(self, description)
+        self.valid_methods = valid_methods
+
+    def get_headers(self, environ=None):
+        headers = HTTPException.get_headers(self, environ)
+        if self.valid_methods:
+            headers.append(("Allow", ", ".join(self.valid_methods)))
+        return headers
+
+
+class NotAcceptable(HTTPException):
+    """*406* `Not Acceptable`
+
+    Raise if the server can't return any content conforming to the
+    `Accept` headers of the client.
+    """
+
+    code = 406
+
+    description = (
+        "The resource identified by the request is only capable of"
+        " generating response entities which have content"
+        " characteristics not acceptable according to the accept"
+        " headers sent in the request."
+    )
+
+
+class RequestTimeout(HTTPException):
+    """*408* `Request Timeout`
+
+    Raise to signalize a timeout.
+    """
+
+    code = 408
+    description = (
+        "The server closed the network connection because the browser"
+        " didn't finish the request within the specified time."
+    )
+
+
+class Conflict(HTTPException):
+    """*409* `Conflict`
+
+    Raise to signal that a request cannot be completed because it conflicts
+    with the current state on the server.
+
+    .. versionadded:: 0.7
+    """
+
+    code = 409
+    description = (
+        "A conflict happened while processing the request. The"
+        " resource might have been modified while the request was being"
+        " processed."
+    )
+
+
+class Gone(HTTPException):
+    """*410* `Gone`
+
+    Raise if a resource existed previously and went away without new location.
+    """
+
+    code = 410
+    description = (
+        "The requested URL is no longer available on this server and"
+        " there is no forwarding address. If you followed a link from a"
+        " foreign page, please contact the author of this page."
+    )
+
+
+class LengthRequired(HTTPException):
+    """*411* `Length Required`
+
+    Raise if the browser submitted data but no ``Content-Length`` header which
+    is required for the kind of processing the server does.
+    """
+
+    code = 411
+    description = (
+        "A request with this method requires a valid <code>Content-"
+        "Length</code> header."
+    )
+
+
+class PreconditionFailed(HTTPException):
+    """*412* `Precondition Failed`
+
+    Status code used in combination with ``If-Match``, ``If-None-Match``, or
+    ``If-Unmodified-Since``.
+    """
+
+    code = 412
+    description = (
+        "The precondition on the request for the URL failed positive evaluation."
+    )
+
+
+class RequestEntityTooLarge(HTTPException):
+    """*413* `Request Entity Too Large`
+
+    The status code one should return if the data submitted exceeded a given
+    limit.
+    """
+
+    code = 413
+    description = "The data value transmitted exceeds the capacity limit."
+
+
+class RequestURITooLarge(HTTPException):
+    """*414* `Request URI Too Large`
+
+    Like *413* but for too long URLs.
+    """
+
+    code = 414
+    description = (
+        "The length of the requested URL exceeds the capacity limit for"
+        " this server. The request cannot be processed."
+    )
+
+
+class UnsupportedMediaType(HTTPException):
+    """*415* `Unsupported Media Type`
+
+    The status code returned if the server is unable to handle the media type
+    the client transmitted.
+    """
+
+    code = 415
+    description = (
+        "The server does not support the media type transmitted in the request."
+    )
+
+
+class RequestedRangeNotSatisfiable(HTTPException):
+    """*416* `Requested Range Not Satisfiable`
+
+    The client asked for an invalid part of the file.
+
+    .. versionadded:: 0.7
+    """
+
+    code = 416
+    description = "The server cannot provide the requested range."
+
+    def __init__(self, length=None, units="bytes", description=None):
+        """Takes an optional `Content-Range` header value based on ``length``
+        parameter.
+        """
+        HTTPException.__init__(self, description)
+        self.length = length
+        self.units = units
+
+    def get_headers(self, environ=None):
+        headers = HTTPException.get_headers(self, environ)
+        if self.length is not None:
+            headers.append(("Content-Range", "%s */%d" % (self.units, self.length)))
+        return headers
+
+
+class ExpectationFailed(HTTPException):
+    """*417* `Expectation Failed`
+
+    The server cannot meet the requirements of the Expect request-header.
+
+    .. versionadded:: 0.7
+    """
+
+    code = 417
+    description = "The server could not meet the requirements of the Expect header"
+
+
+class ImATeapot(HTTPException):
+    """*418* `I'm a teapot`
+
+    The server should return this if it is a teapot and someone attempted
+    to brew coffee with it.
+
+    .. versionadded:: 0.7
+    """
+
+    code = 418
+    description = "This server is a teapot, not a coffee machine"
+
+
+class UnprocessableEntity(HTTPException):
+    """*422* `Unprocessable Entity`
+
+    Used if the request is well formed, but the instructions are otherwise
+    incorrect.
+    """
+
+    code = 422
+    description = (
+        "The request was well-formed but was unable to be followed due"
+        " to semantic errors."
+    )
+
+
+class Locked(HTTPException):
+    """*423* `Locked`
+
+    Used if the resource that is being accessed is locked.
+    """
+
+    code = 423
+    description = "The resource that is being accessed is locked."
+
+
+class FailedDependency(HTTPException):
+    """*424* `Failed Dependency`
+
+    Used if the method could not be performed on the resource
+    because the requested action depended on another action and that action failed.
+    """
+
+    code = 424
+    description = (
+        "The method could not be performed on the resource because the"
+        " requested action depended on another action and that action"
+        " failed."
+    )
+
+
+class PreconditionRequired(HTTPException):
+    """*428* `Precondition Required`
+
+    The server requires this request to be conditional, typically to prevent
+    the lost update problem, which is a race condition between two or more
+    clients attempting to update a resource through PUT or DELETE. By requiring
+    each client to include a conditional header ("If-Match" or "If-Unmodified-
+    Since") with the proper value retained from a recent GET request, the
+    server ensures that each client has at least seen the previous revision of
+    the resource.
+    """
+
+    code = 428
+    description = (
+        "This request is required to be conditional; try using"
+        ' "If-Match" or "If-Unmodified-Since".'
+    )
+
+
+class TooManyRequests(HTTPException):
+    """*429* `Too Many Requests`
+
+    The server is limiting the rate at which this user receives responses, and
+    this request exceeds that rate. (The server may use any convenient method
+    to identify users and their request rates). The server may include a
+    "Retry-After" header to indicate how long the user should wait before
+    retrying.
+    """
+
+    code = 429
+    description = "This user has exceeded an allotted request count. Try again later."
+
+
+class RequestHeaderFieldsTooLarge(HTTPException):
+    """*431* `Request Header Fields Too Large`
+
+    The server refuses to process the request because the header fields are too
+    large. One or more individual fields may be too large, or the set of all
+    headers is too large.
+    """
+
+    code = 431
+    description = "One or more header fields exceeds the maximum size."
+
+
+class UnavailableForLegalReasons(HTTPException):
+    """*451* `Unavailable For Legal Reasons`
+
+    This status code indicates that the server is denying access to the
+    resource as a consequence of a legal demand.
+    """
+
+    code = 451
+    description = "Unavailable for legal reasons."
+
+
+class InternalServerError(HTTPException):
+    """*500* `Internal Server Error`
+
+    Raise if an internal server error occurred.  This is a good fallback if an
+    unknown error occurred in the dispatcher.
+    """
+
+    code = 500
+    description = (
+        "The server encountered an internal error and was unable to"
+        " complete your request. Either the server is overloaded or"
+        " there is an error in the application."
+    )
+
+
+class NotImplemented(HTTPException):
+    """*501* `Not Implemented`
+
+    Raise if the application does not support the action requested by the
+    browser.
+    """
+
+    code = 501
+    description = "The server does not support the action requested by the browser."
+
+
+class BadGateway(HTTPException):
+    """*502* `Bad Gateway`
+
+    If you do proxying in your application you should return this status code
+    if you received an invalid response from the upstream server it accessed
+    in attempting to fulfill the request.
+    """
+
+    code = 502
+    description = (
+        "The proxy server received an invalid response from an upstream server."
+    )
+
+
+class ServiceUnavailable(HTTPException):
+    """*503* `Service Unavailable`
+
+    Status code you should return if a service is temporarily unavailable.
+    """
+
+    code = 503
+    description = (
+        "The server is temporarily unable to service your request due"
+        " to maintenance downtime or capacity problems. Please try"
+        " again later."
+    )
+
+
+class GatewayTimeout(HTTPException):
+    """*504* `Gateway Timeout`
+
+    Status code you should return if a connection to an upstream server
+    times out.
+    """
+
+    code = 504
+    description = "The connection to an upstream server timed out."
+
+
+class HTTPVersionNotSupported(HTTPException):
+    """*505* `HTTP Version Not Supported`
+
+    The server does not support the HTTP protocol version used in the request.
+    """
+
+    code = 505
+    description = (
+        "The server does not support the HTTP protocol version used in the request."
+    )
+
+
+default_exceptions = {}
+__all__ = ["HTTPException"]
+
+
+def _find_exceptions():
+    for _name, obj in iteritems(globals()):
+        try:
+            is_http_exception = issubclass(obj, HTTPException)
+        except TypeError:
+            is_http_exception = False
+        if not is_http_exception or obj.code is None:
+            continue
+        __all__.append(obj.__name__)
+        old_obj = default_exceptions.get(obj.code, None)
+        if old_obj is not None and issubclass(obj, old_obj):
+            continue
+        default_exceptions[obj.code] = obj
+
+
+_find_exceptions()
+del _find_exceptions
+
+
+class Aborter(object):
+    """When passed a dict of code -> exception items it can be used as
+    callable that raises exceptions.  If the first argument to the
+    callable is an integer it will be looked up in the mapping, if it's
+    a WSGI application it will be raised in a proxy exception.
+
+    The rest of the arguments are forwarded to the exception constructor.
+    """
+
+    def __init__(self, mapping=None, extra=None):
+        if mapping is None:
+            mapping = default_exceptions
+        self.mapping = dict(mapping)
+        if extra is not None:
+            self.mapping.update(extra)
+
+    def __call__(self, code, *args, **kwargs):
+        if not args and not kwargs and not isinstance(code, integer_types):
+            raise HTTPException(response=code)
+        if code not in self.mapping:
+            raise LookupError("no exception for %r" % code)
+        raise self.mapping[code](*args, **kwargs)
+
+
+def abort(status, *args, **kwargs):
+    """Raises an :py:exc:`HTTPException` for the given status code or WSGI
+    application::
+
+        abort(404)  # 404 Not Found
+        abort(Response('Hello World'))
+
+    Can be passed a WSGI application or a status code.  If a status code is
+    given it's looked up in the list of exceptions and will raise that
+    exception, if passed a WSGI application it will wrap it in a proxy WSGI
+    exception and raise that::
+
+       abort(404)
+       abort(Response('Hello World'))
+
+    """
+    return _aborter(status, *args, **kwargs)
+
+
+_aborter = Aborter()
+
+
+#: an exception that is used internally to signal both a key error and a
+#: bad request.  Used by a lot of the datastructures.
+BadRequestKeyError = BadRequest.wrap(KeyError)
+
+# imported here because of circular dependencies of werkzeug.utils
+from .http import HTTP_STATUS_CODES
+from .utils import escape

python/werkzeug/filesystem.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.filesystem
+    ~~~~~~~~~~~~~~~~~~~
+
+    Various utilities for the local filesystem.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import codecs
+import sys
+import warnings
+
+# We do not trust traditional unixes.
+has_likely_buggy_unicode_filesystem = (
+    sys.platform.startswith("linux") or "bsd" in sys.platform
+)
+
+
+def _is_ascii_encoding(encoding):
+    """Given an encoding this figures out if the encoding is actually ASCII (which
+    is something we don't actually want in most cases). This is necessary
+    because ASCII comes under many names such as ANSI_X3.4-1968.
+    """
+    if encoding is None:
+        return False
+    try:
+        return codecs.lookup(encoding).name == "ascii"
+    except LookupError:
+        return False
+
+
+class BrokenFilesystemWarning(RuntimeWarning, UnicodeWarning):
+    """The warning used by Werkzeug to signal a broken filesystem. Will only be
+    used once per runtime."""
+
+
+_warned_about_filesystem_encoding = False
+
+
+def get_filesystem_encoding():
+    """Returns the filesystem encoding that should be used. Note that this is
+    different from the Python understanding of the filesystem encoding which
+    might be deeply flawed. Do not use this value against Python's unicode APIs
+    because it might be different. See :ref:`filesystem-encoding` for the exact
+    behavior.
+
+    The concept of a filesystem encoding in generally is not something you
+    should rely on. As such if you ever need to use this function except for
+    writing wrapper code reconsider.
+    """
+    global _warned_about_filesystem_encoding
+    rv = sys.getfilesystemencoding()
+    if has_likely_buggy_unicode_filesystem and not rv or _is_ascii_encoding(rv):
+        if not _warned_about_filesystem_encoding:
+            warnings.warn(
+                "Detected a misconfigured UNIX filesystem: Will use"
+                " UTF-8 as filesystem encoding instead of {0!r}".format(rv),
+                BrokenFilesystemWarning,
+            )
+            _warned_about_filesystem_encoding = True
+        return "utf-8"
+    return rv

python/werkzeug/formparser.py

@@ -0,0 +1,586 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.formparser
+    ~~~~~~~~~~~~~~~~~~~
+
+    This module implements the form parsing.  It supports url-encoded forms
+    as well as non-nested multipart uploads.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import codecs
+import re
+from functools import update_wrapper
+from itertools import chain
+from itertools import repeat
+from itertools import tee
+
+from ._compat import BytesIO
+from ._compat import text_type
+from ._compat import to_native
+from .datastructures import FileStorage
+from .datastructures import Headers
+from .datastructures import MultiDict
+from .http import parse_options_header
+from .urls import url_decode_stream
+from .wsgi import get_content_length
+from .wsgi import get_input_stream
+from .wsgi import make_line_iter
+
+# there are some platforms where SpooledTemporaryFile is not available.
+# In that case we need to provide a fallback.
+try:
+    from tempfile import SpooledTemporaryFile
+except ImportError:
+    from tempfile import TemporaryFile
+
+    SpooledTemporaryFile = None
+
+
+#: an iterator that yields empty strings
+_empty_string_iter = repeat("")
+
+#: a regular expression for multipart boundaries
+_multipart_boundary_re = re.compile("^[ -~]{0,200}[!-~]$")
+
+#: supported http encodings that are also available in python we support
+#: for multipart messages.
+_supported_multipart_encodings = frozenset(["base64", "quoted-printable"])
+
+
+def default_stream_factory(
+    total_content_length, filename, content_type, content_length=None
+):
+    """The stream factory that is used per default."""
+    max_size = 1024 * 500
+    if SpooledTemporaryFile is not None:
+        return SpooledTemporaryFile(max_size=max_size, mode="wb+")
+    if total_content_length is None or total_content_length > max_size:
+        return TemporaryFile("wb+")
+    return BytesIO()
+
+
+def parse_form_data(
+    environ,
+    stream_factory=None,
+    charset="utf-8",
+    errors="replace",
+    max_form_memory_size=None,
+    max_content_length=None,
+    cls=None,
+    silent=True,
+):
+    """Parse the form data in the environ and return it as tuple in the form
+    ``(stream, form, files)``.  You should only call this method if the
+    transport method is `POST`, `PUT`, or `PATCH`.
+
+    If the mimetype of the data transmitted is `multipart/form-data` the
+    files multidict will be filled with `FileStorage` objects.  If the
+    mimetype is unknown the input stream is wrapped and returned as first
+    argument, else the stream is empty.
+
+    This is a shortcut for the common usage of :class:`FormDataParser`.
+
+    Have a look at :ref:`dealing-with-request-data` for more details.
+
+    .. versionadded:: 0.5
+       The `max_form_memory_size`, `max_content_length` and
+       `cls` parameters were added.
+
+    .. versionadded:: 0.5.1
+       The optional `silent` flag was added.
+
+    :param environ: the WSGI environment to be used for parsing.
+    :param stream_factory: An optional callable that returns a new read and
+                           writeable file descriptor.  This callable works
+                           the same as :meth:`~BaseResponse._get_file_stream`.
+    :param charset: The character set for URL and url encoded form data.
+    :param errors: The encoding error behavior.
+    :param max_form_memory_size: the maximum number of bytes to be accepted for
+                           in-memory stored form data.  If the data
+                           exceeds the value specified an
+                           :exc:`~exceptions.RequestEntityTooLarge`
+                           exception is raised.
+    :param max_content_length: If this is provided and the transmitted data
+                               is longer than this value an
+                               :exc:`~exceptions.RequestEntityTooLarge`
+                               exception is raised.
+    :param cls: an optional dict class to use.  If this is not specified
+                       or `None` the default :class:`MultiDict` is used.
+    :param silent: If set to False parsing errors will not be caught.
+    :return: A tuple in the form ``(stream, form, files)``.
+    """
+    return FormDataParser(
+        stream_factory,
+        charset,
+        errors,
+        max_form_memory_size,
+        max_content_length,
+        cls,
+        silent,
+    ).parse_from_environ(environ)
+
+
+def exhaust_stream(f):
+    """Helper decorator for methods that exhausts the stream on return."""
+
+    def wrapper(self, stream, *args, **kwargs):
+        try:
+            return f(self, stream, *args, **kwargs)
+        finally:
+            exhaust = getattr(stream, "exhaust", None)
+            if exhaust is not None:
+                exhaust()
+            else:
+                while 1:
+                    chunk = stream.read(1024 * 64)
+                    if not chunk:
+                        break
+
+    return update_wrapper(wrapper, f)
+
+
+class FormDataParser(object):
+    """This class implements parsing of form data for Werkzeug.  By itself
+    it can parse multipart and url encoded form data.  It can be subclassed
+    and extended but for most mimetypes it is a better idea to use the
+    untouched stream and expose it as separate attributes on a request
+    object.
+
+    .. versionadded:: 0.8
+
+    :param stream_factory: An optional callable that returns a new read and
+                           writeable file descriptor.  This callable works
+                           the same as :meth:`~BaseResponse._get_file_stream`.
+    :param charset: The character set for URL and url encoded form data.
+    :param errors: The encoding error behavior.
+    :param max_form_memory_size: the maximum number of bytes to be accepted for
+                           in-memory stored form data.  If the data
+                           exceeds the value specified an
+                           :exc:`~exceptions.RequestEntityTooLarge`
+                           exception is raised.
+    :param max_content_length: If this is provided and the transmitted data
+                               is longer than this value an
+                               :exc:`~exceptions.RequestEntityTooLarge`
+                               exception is raised.
+    :param cls: an optional dict class to use.  If this is not specified
+                       or `None` the default :class:`MultiDict` is used.
+    :param silent: If set to False parsing errors will not be caught.
+    """
+
+    def __init__(
+        self,
+        stream_factory=None,
+        charset="utf-8",
+        errors="replace",
+        max_form_memory_size=None,
+        max_content_length=None,
+        cls=None,
+        silent=True,
+    ):
+        if stream_factory is None:
+            stream_factory = default_stream_factory
+        self.stream_factory = stream_factory
+        self.charset = charset
+        self.errors = errors
+        self.max_form_memory_size = max_form_memory_size
+        self.max_content_length = max_content_length
+        if cls is None:
+            cls = MultiDict
+        self.cls = cls
+        self.silent = silent
+
+    def get_parse_func(self, mimetype, options):
+        return self.parse_functions.get(mimetype)
+
+    def parse_from_environ(self, environ):
+        """Parses the information from the environment as form data.
+
+        :param environ: the WSGI environment to be used for parsing.
+        :return: A tuple in the form ``(stream, form, files)``.
+        """
+        content_type = environ.get("CONTENT_TYPE", "")
+        content_length = get_content_length(environ)
+        mimetype, options = parse_options_header(content_type)
+        return self.parse(get_input_stream(environ), mimetype, content_length, options)
+
+    def parse(self, stream, mimetype, content_length, options=None):
+        """Parses the information from the given stream, mimetype,
+        content length and mimetype parameters.
+
+        :param stream: an input stream
+        :param mimetype: the mimetype of the data
+        :param content_length: the content length of the incoming data
+        :param options: optional mimetype parameters (used for
+                        the multipart boundary for instance)
+        :return: A tuple in the form ``(stream, form, files)``.
+        """
+        if (
+            self.max_content_length is not None
+            and content_length is not None
+            and content_length > self.max_content_length
+        ):
+            raise exceptions.RequestEntityTooLarge()
+        if options is None:
+            options = {}
+
+        parse_func = self.get_parse_func(mimetype, options)
+        if parse_func is not None:
+            try:
+                return parse_func(self, stream, mimetype, content_length, options)
+            except ValueError:
+                if not self.silent:
+                    raise
+
+        return stream, self.cls(), self.cls()
+
+    @exhaust_stream
+    def _parse_multipart(self, stream, mimetype, content_length, options):
+        parser = MultiPartParser(
+            self.stream_factory,
+            self.charset,
+            self.errors,
+            max_form_memory_size=self.max_form_memory_size,
+            cls=self.cls,
+        )
+        boundary = options.get("boundary")
+        if boundary is None:
+            raise ValueError("Missing boundary")
+        if isinstance(boundary, text_type):
+            boundary = boundary.encode("ascii")
+        form, files = parser.parse(stream, boundary, content_length)
+        return stream, form, files
+
+    @exhaust_stream
+    def _parse_urlencoded(self, stream, mimetype, content_length, options):
+        if (
+            self.max_form_memory_size is not None
+            and content_length is not None
+            and content_length > self.max_form_memory_size
+        ):
+            raise exceptions.RequestEntityTooLarge()
+        form = url_decode_stream(stream, self.charset, errors=self.errors, cls=self.cls)
+        return stream, form, self.cls()
+
+    #: mapping of mimetypes to parsing functions
+    parse_functions = {
+        "multipart/form-data": _parse_multipart,
+        "application/x-www-form-urlencoded": _parse_urlencoded,
+        "application/x-url-encoded": _parse_urlencoded,
+    }
+
+
+def is_valid_multipart_boundary(boundary):
+    """Checks if the string given is a valid multipart boundary."""
+    return _multipart_boundary_re.match(boundary) is not None
+
+
+def _line_parse(line):
+    """Removes line ending characters and returns a tuple (`stripped_line`,
+    `is_terminated`).
+    """
+    if line[-2:] in ["\r\n", b"\r\n"]:
+        return line[:-2], True
+    elif line[-1:] in ["\r", "\n", b"\r", b"\n"]:
+        return line[:-1], True
+    return line, False
+
+
+def parse_multipart_headers(iterable):
+    """Parses multipart headers from an iterable that yields lines (including
+    the trailing newline symbol).  The iterable has to be newline terminated.
+
+    The iterable will stop at the line where the headers ended so it can be
+    further consumed.
+
+    :param iterable: iterable of strings that are newline terminated
+    """
+    result = []
+    for line in iterable:
+        line = to_native(line)
+        line, line_terminated = _line_parse(line)
+        if not line_terminated:
+            raise ValueError("unexpected end of line in multipart header")
+        if not line:
+            break
+        elif line[0] in " \t" and result:
+            key, value = result[-1]
+            result[-1] = (key, value + "\n " + line[1:])
+        else:
+            parts = line.split(":", 1)
+            if len(parts) == 2:
+                result.append((parts[0].strip(), parts[1].strip()))
+
+    # we link the list to the headers, no need to create a copy, the
+    # list was not shared anyways.
+    return Headers(result)
+
+
+_begin_form = "begin_form"
+_begin_file = "begin_file"
+_cont = "cont"
+_end = "end"
+
+
+class MultiPartParser(object):
+    def __init__(
+        self,
+        stream_factory=None,
+        charset="utf-8",
+        errors="replace",
+        max_form_memory_size=None,
+        cls=None,
+        buffer_size=64 * 1024,
+    ):
+        self.charset = charset
+        self.errors = errors
+        self.max_form_memory_size = max_form_memory_size
+        self.stream_factory = (
+            default_stream_factory if stream_factory is None else stream_factory
+        )
+        self.cls = MultiDict if cls is None else cls
+
+        # make sure the buffer size is divisible by four so that we can base64
+        # decode chunk by chunk
+        assert buffer_size % 4 == 0, "buffer size has to be divisible by 4"
+        # also the buffer size has to be at least 1024 bytes long or long headers
+        # will freak out the system
+        assert buffer_size >= 1024, "buffer size has to be at least 1KB"
+
+        self.buffer_size = buffer_size
+
+    def _fix_ie_filename(self, filename):
+        """Internet Explorer 6 transmits the full file name if a file is
+        uploaded.  This function strips the full path if it thinks the
+        filename is Windows-like absolute.
+        """
+        if filename[1:3] == ":\\" or filename[:2] == "\\\\":
+            return filename.split("\\")[-1]
+        return filename
+
+    def _find_terminator(self, iterator):
+        """The terminator might have some additional newlines before it.
+        There is at least one application that sends additional newlines
+        before headers (the python setuptools package).
+        """
+        for line in iterator:
+            if not line:
+                break
+            line = line.strip()
+            if line:
+                return line
+        return b""
+
+    def fail(self, message):
+        raise ValueError(message)
+
+    def get_part_encoding(self, headers):
+        transfer_encoding = headers.get("content-transfer-encoding")
+        if (
+            transfer_encoding is not None
+            and transfer_encoding in _supported_multipart_encodings
+        ):
+            return transfer_encoding
+
+    def get_part_charset(self, headers):
+        # Figure out input charset for current part
+        content_type = headers.get("content-type")
+        if content_type:
+            mimetype, ct_params = parse_options_header(content_type)
+            return ct_params.get("charset", self.charset)
+        return self.charset
+
+    def start_file_streaming(self, filename, headers, total_content_length):
+        if isinstance(filename, bytes):
+            filename = filename.decode(self.charset, self.errors)
+        filename = self._fix_ie_filename(filename)
+        content_type = headers.get("content-type")
+        try:
+            content_length = int(headers["content-length"])
+        except (KeyError, ValueError):
+            content_length = 0
+        container = self.stream_factory(
+            total_content_length=total_content_length,
+            filename=filename,
+            content_type=content_type,
+            content_length=content_length,
+        )
+        return filename, container
+
+    def in_memory_threshold_reached(self, bytes):
+        raise exceptions.RequestEntityTooLarge()
+
+    def validate_boundary(self, boundary):
+        if not boundary:
+            self.fail("Missing boundary")
+        if not is_valid_multipart_boundary(boundary):
+            self.fail("Invalid boundary: %s" % boundary)
+        if len(boundary) > self.buffer_size:  # pragma: no cover
+            # this should never happen because we check for a minimum size
+            # of 1024 and boundaries may not be longer than 200.  The only
+            # situation when this happens is for non debug builds where
+            # the assert is skipped.
+            self.fail("Boundary longer than buffer size")
+
+    def parse_lines(self, file, boundary, content_length, cap_at_buffer=True):
+        """Generate parts of
+        ``('begin_form', (headers, name))``
+        ``('begin_file', (headers, name, filename))``
+        ``('cont', bytestring)``
+        ``('end', None)``
+
+        Always obeys the grammar
+        parts = ( begin_form cont* end |
+                  begin_file cont* end )*
+        """
+        next_part = b"--" + boundary
+        last_part = next_part + b"--"
+
+        iterator = chain(
+            make_line_iter(
+                file,
+                limit=content_length,
+                buffer_size=self.buffer_size,
+                cap_at_buffer=cap_at_buffer,
+            ),
+            _empty_string_iter,
+        )
+
+        terminator = self._find_terminator(iterator)
+
+        if terminator == last_part:
+            return
+        elif terminator != next_part:
+            self.fail("Expected boundary at start of multipart data")
+
+        while terminator != last_part:
+            headers = parse_multipart_headers(iterator)
+
+            disposition = headers.get("content-disposition")
+            if disposition is None:
+                self.fail("Missing Content-Disposition header")
+            disposition, extra = parse_options_header(disposition)
+            transfer_encoding = self.get_part_encoding(headers)
+            name = extra.get("name")
+            filename = extra.get("filename")
+
+            # if no content type is given we stream into memory.  A list is
+            # used as a temporary container.
+            if filename is None:
+                yield _begin_form, (headers, name)
+
+            # otherwise we parse the rest of the headers and ask the stream
+            # factory for something we can write in.
+            else:
+                yield _begin_file, (headers, name, filename)
+
+            buf = b""
+            for line in iterator:
+                if not line:
+                    self.fail("unexpected end of stream")
+
+                if line[:2] == b"--":
+                    terminator = line.rstrip()
+                    if terminator in (next_part, last_part):
+                        break
+
+                if transfer_encoding is not None:
+                    if transfer_encoding == "base64":
+                        transfer_encoding = "base64_codec"
+                    try:
+                        line = codecs.decode(line, transfer_encoding)
+                    except Exception:
+                        self.fail("could not decode transfer encoded chunk")
+
+                # we have something in the buffer from the last iteration.
+                # this is usually a newline delimiter.
+                if buf:
+                    yield _cont, buf
+                    buf = b""
+
+                # If the line ends with windows CRLF we write everything except
+                # the last two bytes.  In all other cases however we write
+                # everything except the last byte.  If it was a newline, that's
+                # fine, otherwise it does not matter because we will write it
+                # the next iteration.  this ensures we do not write the
+                # final newline into the stream.  That way we do not have to
+                # truncate the stream.  However we do have to make sure that
+                # if something else than a newline is in there we write it
+                # out.
+                if line[-2:] == b"\r\n":
+                    buf = b"\r\n"
+                    cutoff = -2
+                else:
+                    buf = line[-1:]
+                    cutoff = -1
+                yield _cont, line[:cutoff]
+
+            else:  # pragma: no cover
+                raise ValueError("unexpected end of part")
+
+            # if we have a leftover in the buffer that is not a newline
+            # character we have to flush it, otherwise we will chop of
+            # certain values.
+            if buf not in (b"", b"\r", b"\n", b"\r\n"):
+                yield _cont, buf
+
+            yield _end, None
+
+    def parse_parts(self, file, boundary, content_length):
+        """Generate ``('file', (name, val))`` and
+        ``('form', (name, val))`` parts.
+        """
+        in_memory = 0
+
+        for ellt, ell in self.parse_lines(file, boundary, content_length):
+            if ellt == _begin_file:
+                headers, name, filename = ell
+                is_file = True
+                guard_memory = False
+                filename, container = self.start_file_streaming(
+                    filename, headers, content_length
+                )
+                _write = container.write
+
+            elif ellt == _begin_form:
+                headers, name = ell
+                is_file = False
+                container = []
+                _write = container.append
+                guard_memory = self.max_form_memory_size is not None
+
+            elif ellt == _cont:
+                _write(ell)
+                # if we write into memory and there is a memory size limit we
+                # count the number of bytes in memory and raise an exception if
+                # there is too much data in memory.
+                if guard_memory:
+                    in_memory += len(ell)
+                    if in_memory > self.max_form_memory_size:
+                        self.in_memory_threshold_reached(in_memory)
+
+            elif ellt == _end:
+                if is_file:
+                    container.seek(0)
+                    yield (
+                        "file",
+                        (name, FileStorage(container, filename, name, headers=headers)),
+                    )
+                else:
+                    part_charset = self.get_part_charset(headers)
+                    yield (
+                        "form",
+                        (name, b"".join(container).decode(part_charset, self.errors)),
+                    )
+
+    def parse(self, file, boundary, content_length):
+        formstream, filestream = tee(
+            self.parse_parts(file, boundary, content_length), 2
+        )
+        form = (p[1] for p in formstream if p[0] == "form")
+        files = (p[1] for p in filestream if p[0] == "file")
+        return self.cls(form), self.cls(files)
+
+
+from . import exceptions

python/werkzeug/local.py

@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+"""
+    werkzeug.local
+    ~~~~~~~~~~~~~~
+
+    This module implements context-local objects.
+
+    :copyright: 2007 Pallets
+    :license: BSD-3-Clause
+"""
+import copy
+from functools import update_wrapper
+
+from ._compat import implements_bool
+from ._compat import PY2
+from .wsgi import ClosingIterator
+
+# since each thread has its own greenlet we can just use those as identifiers
+# for the context.  If greenlets are not available we fall back to the
+# current thread ident depending on where it is.
+try:
+    from greenlet import getcurrent as get_ident
+except ImportError:
+    try:
+        from thread import get_ident
+    except ImportError:
+        from _thread import get_ident
+
+
+def release_local(local):
+    """Releases the contents of the local for the current context.
+    This makes it possible to use locals without a manager.
+
+    Example::
+
+        >>> loc = Local()
+        >>> loc.foo = 42
+        >>> release_local(loc)
+        >>> hasattr(loc, 'foo')
+        False
+
+    With this function one can release :class:`Local` objects as well
+    as :class:`LocalStack` objects.  However it is not possible to
+    release data held by proxies that way, one always has to retain
+    a reference to the underlying local object in order to be able
+    to release it.
+
+    .. versionadded:: 0.6.1
+    """
+    local.__release_local__()
+
+
+class Local(object):
+    __slots__ = ("__storage__", "__ident_func__")
+
+    def __init__(self):
+        object.__setattr__(self, "__storage__", {})
+        object.__setattr__(self, "__ident_func__", get_ident)
+
+    def __iter__(self):
+        return iter(self.__storage__.items())
+
+    def __call__(self, proxy):
+        """Create a proxy for a name."""
+        return LocalProxy(self, proxy)
+
+    def __release_local__(self):
+        self.__storage__.pop(self.__ident_func__(), None)
+
+    def __getattr__(self, name):
+        try:
+            return self.__storage__[self.__ident_func__()][name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __setattr__(self, name, value):
+        ident = self.__ident_func__()
+        storage = self.__storage__
+        try:
+            storage[ident][name] = value
+        except KeyError:
+            storage[ident] = {name: value}
+
+    def __delattr__(self, name):
+        try:
+            del self.__storage__[self.__ident_func__()][name]
+        except KeyError:
+            raise AttributeError(name)
+
+
+class LocalStack(object):
+    """This class works similar to a :class:`Local` but keeps a stack
+    of objects instead.  This is best explained with an example::
+
+        >>> ls = LocalStack()
+        >>> ls.push(42)
+        >>> ls.top
+        42
+        >>> ls.push(23)
+        >>> ls.top
+        23
+        >>> ls.pop()
+        23
+        >>> ls.top
+        42
+
+    They can be force released by using a :class:`LocalManager` or with
+    the :func:`release_local` function but the correct way is to pop the
+    item from the stack after using.  When the stack is empty it will
+    no longer be bound to the current context (and as such released).
+
+    By calling the stack without arguments it returns a proxy that resolves to
+    the topmost item on the stack.
+
+    .. versionadded:: 0.6.1
+    """
+
+    def __init__(self):
+        self._local = Local()
+
+    def __release_local__(self):
+        self._local.__release_local__()
+
+    def _get__ident_func__(self):
+        return self._local.__ident_func__
+
+    def _set__ident_func__(self, value):
+        object.__setattr__(self._local, "__ident_func__", value)
+
+    __ident_func__ = property(_get__ident_func__, _set__ident_func__)
+    del _get__ident_func__, _set__ident_func__
+
+    def __call__(self):
+        def _lookup():
+            rv = self.top
+            if rv is None:
+                raise RuntimeError("object unbound")
+            return rv
+
+        return LocalProxy(_lookup)
+
+    def push(self, obj):
+        """Pushes a new item to the stack"""
+        rv = getattr(self._local, "stack", None)
+        if rv is None:
+            self._local.stack = rv = []
+        rv.append(obj)
+        return rv
+
+    def pop(self):
+        """Removes the topmost item from the stack, will return the
+        old value or `None` if the stack was already empty.
+        """
+        stack = getattr(self._local, "stack", None)
+        if stack is None:
+            return None
+        elif len(stack) == 1:
+            release_local(self._local)
+            return stack[-1]
+        else:
+            return stack.pop()
+
+    @property
+    def top(self):
+        """The topmost item on the stack.  If the stack is empty,
+        `None` is returned.
+        """
+        try:
+            return self._local.stack[-1]
+        except (AttributeError, IndexError):
+            return None
+
+
+class LocalManager(object):
+    """Local objects cannot manage themselves. For that you need a local
+    manager.  You can pass a local manager multiple locals or add them later
+    by appending them to `manager.locals`.  Every time the manager cleans up,
+    it will clean up all the data left in the locals for this context.
+
+    The `ident_func` parameter can be added to override the default ident
+    function for the wrapped locals.
+
+    .. versionchanged:: 0.6.1
+       Instead of a manager the :func:`release_local` function can be used
+       as well.
+
+    .. versionchanged:: 0.7
+       `ident_func` was added.
+    """
+
+    def __init__(self, locals=None, ident_func=None):
+        if locals is None:
+            self.locals = []
+        elif isinstance(locals, Local):
+            self.locals = [locals]
+        else:
+            self.locals = list(locals)
+        if ident_func is not None:
+            self.ident_func = ident_func
+            for local in self.locals:
+                object.__setattr__(local, "__ident_func__", ident_func)
+        else:
+            self.ident_func = get_ident
+
+    def get_ident(self):
+        """Return the context identifier the local objects use internally for
+        this context.  You cannot override this method to change the behavior
+        but use it to link other context local objects (such as SQLAlchemy's
+        scoped sessions) to the Werkzeug locals.
+
+        .. versionchanged:: 0.7
+           You can pass a different ident function to the local manager that
+           will then be propagated to all the locals passed to the
+           constructor.
+        """
+        return self.ident_func()
+
+    def cleanup(self):
+        """Manually clean up the data in the locals for this context.  Call
+        this at the end of the request or use `make_middleware()`.
+        """
+        for local in self.locals:
+            release_local(local)
+
+    def make_middleware(self, app):
+        """Wrap a WSGI application so that cleaning up happens after
+        request end.
+        """
+
+        def application(environ, start_response):
+            return ClosingIterator(app(environ, start_response), self.cleanup)
+
+        return application
+
+    def middleware(self, func):
+        """Like `make_middleware` but for decorating functions.
+
+        Example usage::
+
+            @manager.middleware
+            def application(environ, start_response):
+                ...
+
+        The difference to `make_middleware` is that the function passed
+        will have all the arguments copied from the inner application
+        (name, docstring, module).
+        """
+        return update_wrapper(self.make_middleware(func), func)
+
+    def __repr__(self):
+        return "<%s storages: %d>" % (self.__class__.__name__, len(self.locals))
+
+
+@implements_bool
+class LocalProxy(object):
+    """Acts as a proxy for a werkzeug local.  Forwards all operations to
+    a proxied object.  The only operations not supported for forwarding
+    are right handed operands and any kind of assignment.
+
+    Example usage::
+
+        from werkzeug.local import Local
+        l = Local()
+
+        # these are proxies
+        request = l('request')
+        user = l('user')
+
+
+        from werkzeug.local import LocalStack
+        _response_local = LocalStack()
+
+        # this is a proxy
+        response = _response_local()
+
+    Whenever something is bound to l.user / l.request the proxy objects
+    will forward all operations.  If no object is bound a :exc:`RuntimeError`
+    will be raised.
+
+    To create proxies to :class:`Local` or :class:`LocalStack` objects,
+    call the object as shown above.  If you want to have a proxy to an
+    object looked up by a function, you can (as of Werkzeug 0.6.1) pass
+    a function to the :class:`LocalProxy` constructor::
+
+        session = LocalProxy(lambda: get_current_request().session)
+
+    .. versionchanged:: 0.6.1
+       The class can be instantiated with a callable as well now.
+    """
+
+    __slots__ = ("__local", "__dict__", "__name__", "__wrapped__")
+
+    def __init__(self, local, name=None):
+        object.__setattr__(self, "_LocalProxy__local", local)
+        object.__setattr__(self, "__name__", name)
+        if callable(local) and not hasattr(local, "__release_local__"):
+            # "local" is a callable that is not an instance of Local or
+            # LocalManager: mark it as a wrapped function.
+            object.__setattr__(self, "__wrapped__", local)
+
+    def _get_current_object(self):
+        """Return the current object.  This is useful if you want the real
+        object behind the proxy at a time for performance reasons or because
+        you want to pass the object into a different context.
+        """
+        if not hasattr(self.__local, "__release_local__"):
+            return self.__local()
+        try:
+            return getattr(self.__local, self.__name__)
+        except AttributeError:
+            raise RuntimeError("no object bound to %s" % self.__name__)
+
+    @property
+    def __dict__(self):
+        try:
+            return self._get_current_object().__dict__
+        except RuntimeError:
+            raise AttributeError("__dict__")
+
+    def __repr__(self):
+        try:
+            obj = self._get_current_object()
+        except RuntimeError:
+            return "<%s unbound>" % self.__class__.__name__
+        return repr(obj)
+
+    def __bool__(self):
+        try:
+            return bool(self._get_current_object())
+        except RuntimeError:
+            return False
+
+    def __unicode__(self):
+        try:
+            return unicode(self._get_current_object())  # noqa
+        except RuntimeError:
+            return repr(self)
+
+    def __dir__(self):
+        try:
+            return dir(self._get_current_object())
+        except RuntimeError:
+            return []
+
+    def __getattr__(self, name):
+        if name == "__members__":
+            return dir(self._get_current_object())
+        return getattr(self._get_current_object(), name)
+
+    def __setitem__(self, key, value):
+        self._get_current_object()[key] = value
+
+    def __delitem__(self, key):
+        del self._get_current_object()[key]
+
+    if PY2:
+        __getslice__ = lambda x, i, j: x._get_current_object()[i:j]
+
+        def __setslice__(self, i, j, seq):
+            self._get_current_object()[i:j] = seq
+
+        def __delslice__(self, i, j):
+            del self._get_current_object()[i:j]
+
+    __setattr__ = lambda x, n, v: setattr(x._get_current_object(), n, v)
+    __delattr__ = lambda x, n: delattr(x._get_current_object(), n)
+    __str__ = lambda x: str(x._get_current_object())
+    __lt__ = lambda x, o: x._get_current_object() < o
+    __le__ = lambda x, o: x._get_current_object() <= o
+    __eq__ = lambda x, o: x._get_current_object() == o
+    __ne__ = lambda x, o: x._get_current_object() != o
+    __gt__ = lambda x, o: x._get_current_object() > o
+    __ge__ = lambda x, o: x._get_current_object() >= o
+    __cmp__ = lambda x, o: cmp(x._get_current_object(), o)  # noqa
+    __hash__ = lambda x: hash(x._get_current_object())
+    __call__ = lambda x, *a, **kw: x._get_current_object()(*a, **kw)
+    __len__ = lambda x: len(x._get_current_object())
+    __getitem__ = lambda x, i: x._get_current_object()[i]
+    __iter__ = lambda x: iter(x._get_current_object())
+    __contains__ = lambda x, i: i in x._get_current_object()
+    __add__ = lambda x, o: x._get_current_object() + o
+    __sub__ = lambda x, o: x._get_current_object() - o
+    __mul__ = lambda x, o: x._get_current_object() * o
+    __floordiv__ = lambda x, o: x._get_current_object() // o
+    __mod__ = lambda x, o: x._get_current_object() % o
+    __divmod__ = lambda x, o: x._get_current_object().__divmod__(o)
+    __pow__ = lambda x, o: x._get_current_object() ** o
+    __lshift__ = lambda x, o: x._get_current_object() << o
+    __rshift__ = lambda x, o: x._get_current_object() >> o
+    __and__ = lambda x, o: x._get_current_object() & o
+    __xor__ = lambda x, o: x._get_current_object() ^ o
+    __or__ = lambda x, o: x._get_current_object() | o
+    __div__ = lambda x, o: x._get_current_object().__div__(o)
+    __truediv__ = lambda x, o: x._get_current_object().__truediv__(o)
+    __neg__ = lambda x: -(x._get_current_object())
+    __pos__ = lambda x: +(x._get_current_object())
+    __abs__ = lambda x: abs(x._get_current_object())
+    __invert__ = lambda x: ~(x._get_current_object())
+    __complex__ = lambda x: complex(x._get_current_object())
+    __int__ = lambda x: int(x._get_current_object())
+    __long__ = lambda x: long(x._get_current_object())  # noqa
+    __float__ = lambda x: float(x._get_current_object())
+    __oct__ = lambda x: oct(x._get_current_object())
+    __hex__ = lambda x: hex(x._get_current_object())
+    __index__ = lambda x: x._get_current_object().__index__()
+    __coerce__ = lambda x, o: x._get_current_object().__coerce__(x, o)
+    __enter__ = lambda x: x._get_current_object().__enter__()
+    __exit__ = lambda x, *a, **kw: x._get_current_object().__exit__(*a, **kw)
+    __radd__ = lambda x, o: o + x._get_current_object()
+    __rsub__ = lambda x, o: o - x._get_current_object()
+    __rmul__ = lambda x, o: o * x._get_current_object()
+    __rdiv__ = lambda x, o: o / x._get_current_object()
+    if PY2:
+        __rtruediv__ = lambda x, o: x._get_current_object().__rtruediv__(o)
+    else:
+        __rtruediv__ = __rdiv__
+    __rfloordiv__ = lambda x, o: o // x._get_current_object()
+    __rmod__ = lambda x, o: o % x._get_current_object()
+    __rdivmod__ = lambda x, o: x._get_current_object().__rdivmod__(o)
+    __copy__ = lambda x: copy.copy(x._get_current_object())
+    __deepcopy__ = lambda x, memo: copy.deepcopy(x._get_current_object(), memo)