Merge remote-tracking branch 'brett/itsdangerous'

* brett/itsdangerous:
  Call is_updated instead of testing it boolean.
  Harden It's Dangerous key management.
  First tests for the Session class.
  Set a starting value for session.send_new_cookie.
  Remove beaker stuff from the code.
  Delete the session cookie on an empty session.
  Back sessions with It's Dangerous.
  Improve fs security for itsdangerous secret.
  Docs for get_timed_signer_url.
  Basic itsdangerous infrastructure.

Conflicts:
	mediagoblin/tests/test_cache.py

mediagoblin/app.py

@@ -25,7 +25,7 @@ from werkzeug.exceptions import HTTPException
 from werkzeug.routing import RequestRedirect
 
 from mediagoblin import meddleware, __version__
-from mediagoblin.tools import common, translate, template
+from mediagoblin.tools import common, session, translate, template
 from mediagoblin.tools.response import render_http_exception
 from mediagoblin.tools.theme import register_themes
 from mediagoblin.tools import request as mg_request
@@ -34,8 +34,9 @@ from mediagoblin.init.celery import setup_celery_from_config
 from mediagoblin.init.plugins import setup_plugins
 from mediagoblin.init import (get_jinja_loader, get_staticdirector,
     setup_global_and_app_config, setup_locales, setup_workbench, setup_database,
-    setup_storage, setup_beaker_cache)
+    setup_storage)
 from mediagoblin.tools.pluginapi import PluginManager
+from mediagoblin.tools.crypto import setup_crypto
 
 
 _log = logging.getLogger(__name__)
@@ -66,6 +67,8 @@ class MediaGoblinApp(object):
         # Open and setup the config
         global_config, app_config = setup_global_and_app_config(config_path)
 
+        setup_crypto()
+
         ##########################################
         # Setup other connections / useful objects
         ##########################################
@@ -100,9 +103,6 @@ class MediaGoblinApp(object):
         # set up staticdirector tool
         self.staticdirector = get_staticdirector(app_config)
 
-        # set up caching
-        self.cache = setup_beaker_cache()
-
         # Setup celery, if appropriate
         if setup_celery and not app_config.get('celery_setup_elsewhere'):
             if os.environ.get('CELERY_ALWAYS_EAGER', 'false').lower() == 'true':
@@ -157,7 +157,8 @@ class MediaGoblinApp(object):
 
         ## Attach utilities to the request object
         # Do we really want to load this via middleware?  Maybe?
-        request.session = request.environ['beaker.session']
+        session_manager = session.SessionManager()
+        request.session = session_manager.load_session_from_cookie(request)
         # Attach self as request.app
         # Also attach a few utilities from request.app for convenience?
         request.app = self
@@ -226,6 +227,8 @@ class MediaGoblinApp(object):
             response = render_http_exeption(
                 request, e, e.get_description(environ))
 
+        session_manager.save_session_to_cookie(request.session, response)
+
         return response(environ, start_response)
 
     def __call__(self, environ, start_response):

mediagoblin/config_spec.ini

@@ -14,6 +14,9 @@ sql_engine = string(default="sqlite:///%(here)s/mediagoblin.db")
 # Where temporary files used in processing and etc are kept
 workbench_path = string(default="%(here)s/user_dev/media/workbench")
 
+# Where to store cryptographic sensible data
+crypto_path = string(default="%(here)s/user_dev/crypto")
+
 # Where mediagoblin-builtin static assets are kept
 direct_remote_path = string(default="/mgoblin_static/")
 

mediagoblin/init/__init__.py

@@ -14,8 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from beaker.cache import CacheManager
-from beaker.util import parse_cache_config_options
 import jinja2
 
 from mediagoblin.tools import staticdirect
@@ -146,16 +144,3 @@ def setup_workbench():
     workbench_manager = WorkbenchManager(app_config['workbench_path'])
 
     setup_globals(workbench_manager=workbench_manager)
-
-
-def setup_beaker_cache():
-    """
-    Setup the Beaker Cache manager.
-    """
-    cache_config = mg_globals.global_config['beaker.cache']
-    cache_config = dict(
-        [(u'cache.%s' % key, value)
-         for key, value in cache_config.iteritems()])
-    cache = CacheManager(**parse_cache_config_options(cache_config))
-    setup_globals(cache=cache)
-    return cache

mediagoblin/mg_globals.py

@@ -29,9 +29,6 @@ import threading
 # SQL database engine
 database = None
 
-# beaker's cache manager
-cache = None
-
 # should be the same as the 
 public_store = None
 queue_store = None

mediagoblin/tests/test_cache.py

@@ -1,50 +0,0 @@
-# GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-from mediagoblin import mg_globals
-
-
-DATA_TO_CACHE = {
-    'herp': 'derp',
-    'lol': 'cats'}
-
-
-def _get_some_data(key):
-    """
-    Stuid function that makes use of some caching.
-    """
-    some_data_cache = mg_globals.cache.get_cache('sum_data')
-    if some_data_cache.has_key(key):
-        return some_data_cache.get(key)
-
-    value = DATA_TO_CACHE.get(key)
-    some_data_cache.put(key, value)
-    return value
-
-
-def test_cache_working(test_app):
-    some_data_cache = mg_globals.cache.get_cache('sum_data')
-    assert not some_data_cache.has_key('herp')
-    assert _get_some_data('herp') == 'derp'
-    assert some_data_cache.get('herp') == 'derp'
-    # should get the same value again
-    assert _get_some_data('herp') == 'derp'
-
-    # now we force-change it, but the function should use the cached
-    # version
-    some_data_cache.put('herp', 'pred')
-    assert _get_some_data('herp') == 'pred'

mediagoblin/tests/test_session.py

@@ -0,0 +1,30 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from mediagoblin.tools import session
+
+def test_session():
+    sess = session.Session()
+    assert not sess
+    assert not sess.is_updated()
+    sess['user_id'] = 27
+    assert sess
+    assert not sess.is_updated()
+    sess.save()
+    assert sess.is_updated()
+    sess.delete()
+    assert not sess
+    assert sess.is_updated()

mediagoblin/tests/tools.py

@@ -45,9 +45,7 @@ TEST_USER_DEV = pkg_resources.resource_filename(
     'mediagoblin.tests', 'test_user_dev')
 
 
-USER_DEV_DIRECTORIES_TO_SETUP = [
-    'media/public', 'media/queue',
-    'beaker/sessions/data', 'beaker/sessions/lock']
+USER_DEV_DIRECTORIES_TO_SETUP = ['media/public', 'media/queue']
 
 BAD_CELERY_MESSAGE = """\
 Sorry, you *absolutely* must run tests with the

mediagoblin/tools/crypto.py

@@ -0,0 +1,110 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2013 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import errno
+import itsdangerous
+import logging
+import os.path
+import random
+import tempfile
+from mediagoblin import mg_globals
+
+_log = logging.getLogger(__name__)
+
+
+# Use the system (hardware-based) random number generator if it exists.
+# -- this optimization is lifted from Django
+try:
+    getrandbits = random.SystemRandom().getrandbits
+except AttributeError:
+    getrandbits = random.getrandbits
+
+
+__itsda_secret = None
+
+
+def load_key(filename):
+    global __itsda_secret
+    key_file = open(filename)
+    try:
+        __itsda_secret = key_file.read()
+    finally:
+        key_file.close()
+
+def create_key(key_dir, key_filepath):
+    global __itsda_secret
+    old_umask = os.umask(077)
+    key_file = None
+    try:
+        if not os.path.isdir(key_dir):
+            os.makedirs(key_dir)
+            _log.info("Created %s", dirname)
+        key = str(getrandbits(192))
+        key_file = tempfile.NamedTemporaryFile(dir=key_dir, suffix='.bin',
+                                               delete=False)
+        key_file.write(key)
+        key_file.flush()
+        os.rename(key_file.name, key_filepath)
+        key_file.close()
+    finally:
+        os.umask(old_umask)
+        if (key_file is not None) and (not key_file.closed):
+            key_file.close()
+            os.unlink(key_file.name)
+    __itsda_secret = key
+    _log.info("Saved new key for It's Dangerous")
+
+def setup_crypto():
+    global __itsda_secret
+    key_dir = mg_globals.app_config["crypto_path"]
+    key_filepath = os.path.join(key_dir, 'itsdangeroussecret.bin')
+    try:
+        load_key(key_filepath)
+    except IOError, error:
+        if error.errno != errno.ENOENT:
+            raise
+        create_key(key_dir, key_filepath)
+
+def get_timed_signer_url(namespace):
+    """
+    This gives a basic signing/verifying object.
+
+    The namespace makes sure signed tokens can't be used in
+    a different area. Like using a forgot-password-token as
+    a session cookie.
+
+    Basic usage:
+
+    .. code-block:: python
+
+       _signer = None
+       TOKEN_VALID_DAYS = 10
+       def setup():
+           global _signer
+           _signer = get_timed_signer_url("session cookie")
+       def create_token(obj):
+           return _signer.dumps(obj)
+       def parse_token(token):
+           # This might raise an exception in case
+           # of an invalid token, or an expired token.
+           return _signer.loads(token, max_age=TOKEN_VALID_DAYS*24*3600)
+
+    For more details see
+    http://pythonhosted.org/itsdangerous/#itsdangerous.URLSafeTimedSerializer
+    """
+    assert __itsda_secret is not None
+    return itsdangerous.URLSafeTimedSerializer(__itsda_secret,
+         salt=namespace)

mediagoblin/tools/request.py

@@ -35,4 +35,4 @@ def setup_user_in_request(request):
         # Something's wrong... this user doesn't exist?  Invalidate
         # this session.
         _log.warn("Killing session for user id %r", request.session['user_id'])
-        request.session.invalidate()
+        request.session.delete()

mediagoblin/tools/session.py

@@ -0,0 +1,67 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2013 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import itsdangerous
+import logging
+
+import crypto
+
+_log = logging.getLogger(__name__)
+
+class Session(dict):
+    def __init__(self, *args, **kwargs):
+        self.send_new_cookie = False
+        dict.__init__(self, *args, **kwargs)
+
+    def save(self):
+        self.send_new_cookie = True
+
+    def is_updated(self):
+        return self.send_new_cookie
+
+    def delete(self):
+        self.clear()
+        self.save()
+
+
+class SessionManager(object):
+    def __init__(self, cookie_name='MGSession', namespace=None):
+        if namespace is None:
+            namespace = cookie_name
+        self.signer = crypto.get_timed_signer_url(namespace)
+        self.cookie_name = cookie_name
+
+    def load_session_from_cookie(self, request):
+        cookie = request.cookies.get(self.cookie_name)
+        if not cookie:
+            return Session()
+        ### FIXME: Future cookie-blacklisting code
+        # m = BadCookie.query.filter_by(cookie = cookie)
+        # if m:
+        #     _log.warn("Bad cookie received: %s", m.reason)
+        #     raise BadRequest()
+        try:
+            return Session(self.signer.loads(cookie))
+        except itsdangerous.BadData:
+            return Session()
+
+    def save_session_to_cookie(self, session, response):
+        if not session.is_updated():
+            return
+        elif not session:
+            response.delete_cookie(self.cookie_name)
+        else:
+            response.set_cookie(self.cookie_name, self.signer.dumps(session))

paste.ini

@@ -17,7 +17,7 @@ use = egg:Paste#urlmap
 
 [app:mediagoblin]
 use = egg:mediagoblin#app
-filter-with = beaker
+# filter-with = beaker
 config = %(here)s/mediagoblin_local.ini %(here)s/mediagoblin.ini
 
 [loggers]

setup.py

@@ -43,7 +43,6 @@ setup(
     install_requires=[
         'setuptools',
         'PasteScript',
-        'beaker',
         'wtforms',
         'py-bcrypt',
         'pytest',
@@ -61,6 +60,7 @@ setup(
         'sqlalchemy>=0.7.0',
         'sqlalchemy-migrate',
         'mock',
+        'itsdangerous',
         ## This is optional!
         # 'translitcodec',
         ## For now we're expecting that users will install this from