From f1226c98c44119261b6e1a5652d32e49eb912a53 Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sun, 4 Sep 2011 18:15:52 -0700 Subject: [PATCH 01/13] Issue 361 Initial implementation of CSRF protection middleware --- mediagoblin/config_spec.ini | 3 + mediagoblin/middleware/__init__.py | 1 + mediagoblin/middleware/csrf.py | 131 +++++++++++++++++++++++++++++ mediagoblin/util.py | 3 + 4 files changed, 138 insertions(+) create mode 100644 mediagoblin/middleware/csrf.py diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index a0fbde09..8018b243 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -41,6 +41,9 @@ celery_setup_elsewhere = boolean(default=False) # source files for a media file but can also be a HUGE security risk. allow_attachments = boolean(default=False) +# Cookie stuff +secret_key = string(default="Something Super Duper Secrit!") +csrf_cookie_name = string(default='mediagoblin_nonce') [storage:publicstore] base_dir = string(default="%(here)s/user_dev/media/public") diff --git a/mediagoblin/middleware/__init__.py b/mediagoblin/middleware/__init__.py index 586debbf..05325ee5 100644 --- a/mediagoblin/middleware/__init__.py +++ b/mediagoblin/middleware/__init__.py @@ -16,4 +16,5 @@ ENABLED_MIDDLEWARE = ( 'mediagoblin.middleware.noop:NoOpMiddleware', + 'mediagoblin.middleware.csrf:CsrfMiddleware', ) diff --git a/mediagoblin/middleware/csrf.py b/mediagoblin/middleware/csrf.py new file mode 100644 index 00000000..a372d0b5 --- /dev/null +++ b/mediagoblin/middleware/csrf.py @@ -0,0 +1,131 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import random + +from webob.exc import HTTPForbidden +from wtforms import Form, HiddenField, validators + +from mediagoblin import mg_globals + +# Use the system (hardware-based) random number generator if it exists. +# -- this optimization is lifted from Django +if hasattr(random, 'SystemRandom'): + randrange = random.SystemRandom().randrange +else: + randrange = random.randrange + + +class CsrfForm(Form): + """Simple form to handle rendering a CSRF token and confirming it + is included in the POST.""" + + csrf_token = HiddenField("", + [validators.Required()]) + +def render_csrf_form_token(request): + """Render the CSRF token in a format suitable for inclusion in a + form.""" + + form = CsrfForm(csrf_token = request.environ['CSRF_TOKEN']) + + return form.csrf_token + +class CsrfMiddleware(object): + """CSRF Protection Middleware + + Adds a CSRF Cookie to responses and verifies that it is present + and matches the form token for non-safe requests. + """ + + MAX_CSRF_KEY = 2 << 63 + SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS", "TRACE") + + def __init__(self, mg_app): + self.app = mg_app + + def process_request(self, request): + """For non-safe requests, confirm that the tokens are present + and match. + """ + + # get the token from the cookie + try: + request.environ['CSRF_TOKEN'] = \ + request.cookies[mg_globals.app_config['csrf_cookie_name']] + + except KeyError, e: + # if it doesn't exist, make a new one + request.environ['CSRF_TOKEN'] = self._make_token(request) + + # if this is a non-"safe" request (ie, one that could have + # side effects), confirm that the CSRF tokens are present and + # valid + if request.method not in self.SAFE_HTTP_METHODS: + return self.verify_tokens(request) + + def process_response(self, request, response): + """Add the CSRF cookie to the response if needed and set Vary + headers. + """ + + # set the CSRF cookie + response.set_cookie( + mg_globals.app_config['csrf_cookie_name'], + request.environ['CSRF_TOKEN'], + max_age=60*60*24*7*52, path='/', + domain=mg_globals.app_config.get('csrf_cookie_domain', None), + secure=(request.scheme.lower() == 'https'), + httponly=True) + + # update the Vary header + response.vary = (response.vary or []) + ['Cookie'] + + def _make_token(self, request): + """Generate a new token to use for CSRF protection.""" + + return hashlib.md5("%s%s" % + (randrange(0, self.MAX_CSRF_KEY), + mg_globals.app_config['secret_key']) + ).hexdigest() + + def verify_tokens(self, request): + """Verify that the CSRF Cookie exists and that it matches the + form value.""" + + # confirm the cookie token was presented + cookie_token = request.cookies.get( + mg_globals.app_config['csrf_cookie_name'], + None) + + if cookie_token is None: + # the CSRF cookie must be present in the request + return HTTPForbidden() + + # get the form token and confirm it matches + form = CsrfForm(request.POST) + if form.validate(): + form_token = form.csrf_token.data + + if form_token == cookie_token: + # all's well that ends well + return + + # either the tokens didn't match or the form token wasn't + # present; either way, the request is denied + return HTTPForbidden() + diff --git a/mediagoblin/util.py b/mediagoblin/util.py index e391b8b0..bc72f8df 100644 --- a/mediagoblin/util.py +++ b/mediagoblin/util.py @@ -39,6 +39,7 @@ from wtforms.form import Form from mediagoblin import mg_globals from mediagoblin import messages from mediagoblin.db.util import ObjectId +from mediagoblin.middleware.csrf import render_csrf_form_token from itertools import izip, count @@ -125,6 +126,8 @@ def render_template(request, template_path, context): template = request.template_env.get_template( template_path) context['request'] = request + context['csrf_token'] = render_csrf_form_token(request) + rendered = template.render(context) if TESTS_ENABLED: From 0a8a3fc1571100aba3bd3a3dec98f5e9e252780b Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sun, 4 Sep 2011 18:16:03 -0700 Subject: [PATCH 02/13] Issue 361: Include the CSRF token in all forms --- mediagoblin/templates/mediagoblin/auth/login.html | 1 + mediagoblin/templates/mediagoblin/auth/register.html | 1 + mediagoblin/templates/mediagoblin/edit/attachments.html | 1 + mediagoblin/templates/mediagoblin/edit/edit.html | 1 + mediagoblin/templates/mediagoblin/edit/edit_profile.html | 1 + mediagoblin/templates/mediagoblin/submit/start.html | 1 + mediagoblin/templates/mediagoblin/test_submit.html | 1 + mediagoblin/templates/mediagoblin/user_pages/media.html | 1 + .../templates/mediagoblin/user_pages/media_confirm_delete.html | 1 + 9 files changed, 9 insertions(+) diff --git a/mediagoblin/templates/mediagoblin/auth/login.html b/mediagoblin/templates/mediagoblin/auth/login.html index 958cf9ea..1be58560 100644 --- a/mediagoblin/templates/mediagoblin/auth/login.html +++ b/mediagoblin/templates/mediagoblin/auth/login.html @@ -22,6 +22,7 @@ {% block mediagoblin_content %}
+ {{ csrf_token }}

{% trans %}Log in{% endtrans %}

{% if login_failed %} diff --git a/mediagoblin/templates/mediagoblin/auth/register.html b/mediagoblin/templates/mediagoblin/auth/register.html index e72b3a52..25b68058 100644 --- a/mediagoblin/templates/mediagoblin/auth/register.html +++ b/mediagoblin/templates/mediagoblin/auth/register.html @@ -26,6 +26,7 @@

{% trans %}Create an account!{% endtrans %}

{{ wtforms_util.render_divs(register_form) }} + {{ csrf_token }}
diff --git a/mediagoblin/templates/mediagoblin/edit/attachments.html b/mediagoblin/templates/mediagoblin/edit/attachments.html index 63b06581..d8b55f58 100644 --- a/mediagoblin/templates/mediagoblin/edit/attachments.html +++ b/mediagoblin/templates/mediagoblin/edit/attachments.html @@ -49,6 +49,7 @@
Cancel + {{ csrf_token }}
diff --git a/mediagoblin/templates/mediagoblin/edit/edit.html b/mediagoblin/templates/mediagoblin/edit/edit.html index 8c4e2efb..b4b3be85 100644 --- a/mediagoblin/templates/mediagoblin/edit/edit.html +++ b/mediagoblin/templates/mediagoblin/edit/edit.html @@ -35,6 +35,7 @@
{% trans %}Cancel{% endtrans %} + {{ csrf_token }}
diff --git a/mediagoblin/templates/mediagoblin/edit/edit_profile.html b/mediagoblin/templates/mediagoblin/edit/edit_profile.html index 464c663d..93b2a792 100644 --- a/mediagoblin/templates/mediagoblin/edit/edit_profile.html +++ b/mediagoblin/templates/mediagoblin/edit/edit_profile.html @@ -33,6 +33,7 @@ {{ wtforms_util.render_divs(form) }}
+ {{ csrf_token }}
diff --git a/mediagoblin/templates/mediagoblin/submit/start.html b/mediagoblin/templates/mediagoblin/submit/start.html index f2e844df..7bc6ff45 100644 --- a/mediagoblin/templates/mediagoblin/submit/start.html +++ b/mediagoblin/templates/mediagoblin/submit/start.html @@ -26,6 +26,7 @@

{% trans %}Submit yer media{% endtrans %}

{{ wtforms_util.render_divs(submit_form) }}
+ {{ csrf_token }}
diff --git a/mediagoblin/templates/mediagoblin/test_submit.html b/mediagoblin/templates/mediagoblin/test_submit.html index 78b88ae8..190b9ac3 100644 --- a/mediagoblin/templates/mediagoblin/test_submit.html +++ b/mediagoblin/templates/mediagoblin/test_submit.html @@ -26,6 +26,7 @@ + {{ csrf_token }} diff --git a/mediagoblin/templates/mediagoblin/user_pages/media.html b/mediagoblin/templates/mediagoblin/user_pages/media.html index 442bef6d..433f74dc 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media.html @@ -72,6 +72,7 @@ {{ wtforms_util.render_divs(comment_form) }}
+ {{ csrf_token }}
{% endif %} diff --git a/mediagoblin/templates/mediagoblin/user_pages/media_confirm_delete.html b/mediagoblin/templates/mediagoblin/user_pages/media_confirm_delete.html index 48fbc3b0..3acf802b 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media_confirm_delete.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media_confirm_delete.html @@ -42,6 +42,7 @@ {{ wtforms_util.render_divs(form) }}
+ {{ csrf_token }}
From dd1756ee19fc22d13bd207174e37ec5a68892a89 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Thu, 15 Sep 2011 13:19:25 +0200 Subject: [PATCH 03/13] mountstorage - Changed typo in import --- mediagoblin/storage/mountstorage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/storage/mountstorage.py b/mediagoblin/storage/mountstorage.py index 6adb7a0d..7239931f 100644 --- a/mediagoblin/storage/mountstorage.py +++ b/mediagoblin/storage/mountstorage.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from medigoblin.storage import StorageInterface, clean_listy_filepath +from mediagoblin.storage import StorageInterface, clean_listy_filepath class MountStorage(StorageInterface): From 6347605c1e57e2f428b682740048c68cb54837c2 Mon Sep 17 00:00:00 2001 From: Elrond Date: Thu, 29 Sep 2011 20:15:53 +0200 Subject: [PATCH 04/13] When using paste's static content server to serve media entries and other files the client should have some idea on how long it can cache those files locally before asking again for them. The old setting was: Don't allow the client to cache. New setting: 1 week for the media entries (they don't change, ever) 1 day for css/logos, etc. They change on an update, so people might want to see the new design soon. --- paste.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paste.ini b/paste.ini index fc459989..7eee528b 100644 --- a/paste.ini +++ b/paste.ini @@ -19,10 +19,12 @@ config = %(here)s/mediagoblin.ini [app:publicstore_serve] use = egg:Paste#static document_root = %(here)s/user_dev/media/public/ +cache_max_age = 604800 [app:mediagoblin_static] use = egg:Paste#static document_root = %(here)s/mediagoblin/static/ +cache_max_age = 86400 [filter:beaker] use = egg:Beaker#beaker_session From ae3bc7fabf8e0abb5f3d8b6534ca451890bbe90b Mon Sep 17 00:00:00 2001 From: Aaron Williamson Date: Sat, 1 Oct 2011 09:31:42 -0400 Subject: [PATCH 05/13] Moved common, translation, template, and url code out of util.py and into tools/[file].py --- mediagoblin/app.py | 5 +- mediagoblin/auth/forms.py | 2 +- mediagoblin/auth/lib.py | 3 +- mediagoblin/auth/views.py | 2 +- mediagoblin/db/models.py | 4 +- mediagoblin/process_media/errors.py | 2 +- mediagoblin/submit/forms.py | 2 +- mediagoblin/submit/views.py | 2 +- mediagoblin/tests/test_auth.py | 106 +++---- mediagoblin/tests/test_messages.py | 4 +- mediagoblin/tests/test_submission.py | 42 +-- mediagoblin/tests/test_util.py | 36 +-- mediagoblin/tools/__init__.py | 0 mediagoblin/tools/common.py | 18 ++ mediagoblin/tools/template.py | 114 ++++++++ mediagoblin/tools/translate.py | 167 +++++++++++ mediagoblin/tools/url.py | 31 +++ mediagoblin/user_pages/forms.py | 2 +- mediagoblin/user_pages/views.py | 2 +- mediagoblin/util.py | 395 ++++++++++++++------------- 20 files changed, 636 insertions(+), 303 deletions(-) create mode 100644 mediagoblin/tools/__init__.py create mode 100644 mediagoblin/tools/common.py create mode 100644 mediagoblin/tools/template.py create mode 100644 mediagoblin/tools/translate.py create mode 100644 mediagoblin/tools/url.py diff --git a/mediagoblin/app.py b/mediagoblin/app.py index dd5f0b89..5ee3b973 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -21,6 +21,7 @@ import routes from webob import Request, exc from mediagoblin import routing, util, middleware +from mediagoblin.tools import translate, template from mediagoblin.mg_globals import setup_globals from mediagoblin.init.celery import setup_celery_from_config from mediagoblin.init import (get_jinja_loader, get_staticdirector, @@ -123,9 +124,9 @@ class MediaGoblinApp(object): # Attach self as request.app # Also attach a few utilities from request.app for convenience? request.app = self - request.locale = util.get_locale_from_request(request) + request.locale = translate.get_locale_from_request(request) - request.template_env = util.get_jinja_env( + request.template_env = template.get_jinja_env( self.template_loader, request.locale) request.db = self.db request.staticdirect = self.staticdirector diff --git a/mediagoblin/auth/forms.py b/mediagoblin/auth/forms.py index 6339b4a3..a932ad26 100644 --- a/mediagoblin/auth/forms.py +++ b/mediagoblin/auth/forms.py @@ -17,7 +17,7 @@ import wtforms import re -from mediagoblin.util import fake_ugettext_passthrough as _ +from mediagoblin.tools.translate import fake_ugettext_passthrough as _ class RegistrationForm(wtforms.Form): diff --git a/mediagoblin/auth/lib.py b/mediagoblin/auth/lib.py index d7d351a5..bf5a2399 100644 --- a/mediagoblin/auth/lib.py +++ b/mediagoblin/auth/lib.py @@ -19,7 +19,8 @@ import random import bcrypt -from mediagoblin.util import send_email, render_template +from mediagoblin.util import send_email +from mediagoblin.tools.template import render_template from mediagoblin import mg_globals diff --git a/mediagoblin/auth/views.py b/mediagoblin/auth/views.py index f67f0588..9bfa93cf 100644 --- a/mediagoblin/auth/views.py +++ b/mediagoblin/auth/views.py @@ -22,7 +22,7 @@ from webob import exc from mediagoblin import messages from mediagoblin import mg_globals from mediagoblin.util import render_to_response, redirect, render_404 -from mediagoblin.util import pass_to_ugettext as _ +from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.db.util import ObjectId, InvalidId from mediagoblin.auth import lib as auth_lib from mediagoblin.auth import forms as auth_forms diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index bbddada6..eacc801c 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -25,7 +25,7 @@ from mediagoblin.db import migrations from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId from mediagoblin.util import Pagination from mediagoblin.util import DISPLAY_IMAGE_FETCHING_ORDER - +from mediagoblin.tools import url ################### # Custom validators @@ -242,7 +242,7 @@ class MediaEntry(Document): pass def generate_slug(self): - self['slug'] = util.slugify(self['title']) + self['slug'] = url.slugify(self['title']) duplicate = mg_globals.database.media_entries.find_one( {'slug': self['slug']}) diff --git a/mediagoblin/process_media/errors.py b/mediagoblin/process_media/errors.py index 156f0a01..8003ffaf 100644 --- a/mediagoblin/process_media/errors.py +++ b/mediagoblin/process_media/errors.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mediagoblin.util import lazy_pass_to_ugettext as _ +from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ class BaseProcessingFail(Exception): """ diff --git a/mediagoblin/submit/forms.py b/mediagoblin/submit/forms.py index a999c714..200ce4e4 100644 --- a/mediagoblin/submit/forms.py +++ b/mediagoblin/submit/forms.py @@ -18,7 +18,7 @@ import wtforms from mediagoblin.util import tag_length_validator -from mediagoblin.util import fake_ugettext_passthrough as _ +from mediagoblin.tools.translate import fake_ugettext_passthrough as _ class SubmitStartForm(wtforms.Form): diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index e24d78f3..cd34e006 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -25,7 +25,7 @@ from mediagoblin.db.util import ObjectId from mediagoblin.util import ( render_to_response, redirect, cleaned_markdown_conversion, \ convert_to_tag_list_of_dicts) -from mediagoblin.util import pass_to_ugettext as _ +from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.decorators import require_active_login from mediagoblin.submit import forms as submit_forms, security from mediagoblin.process_media import process_media, mark_entry_failed diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index fbbe1613..f00456c4 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -22,7 +22,7 @@ from nose.tools import assert_equal from mediagoblin.auth import lib as auth_lib from mediagoblin.tests.tools import setup_fresh_app from mediagoblin import mg_globals -from mediagoblin import util +from mediagoblin.tools import template ######################## @@ -76,16 +76,16 @@ def test_register_views(test_app): test_app.get('/auth/register/') # Make sure it rendered with the appropriate template - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/auth/register.html') # Try to register without providing anything, should error # -------------------------------------------------------- - util.clear_test_template_context() + template.clear_test_template_context() test_app.post( '/auth/register/', {}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] form = context['register_form'] assert form.username.errors == [u'This field is required.'] assert form.password.errors == [u'This field is required.'] @@ -96,14 +96,14 @@ def test_register_views(test_app): # -------------------------------------------------------- ## too short - util.clear_test_template_context() + template.clear_test_template_context() test_app.post( '/auth/register/', { 'username': 'l', 'password': 'o', 'confirm_password': 'o', 'email': 'l'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] form = context['register_form'] assert form.username.errors == [ @@ -112,12 +112,12 @@ def test_register_views(test_app): u'Field must be between 6 and 30 characters long.'] ## bad form - util.clear_test_template_context() + template.clear_test_template_context() test_app.post( '/auth/register/', { 'username': '@_@', 'email': 'lollerskates'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] form = context['register_form'] assert form.username.errors == [ @@ -126,12 +126,12 @@ def test_register_views(test_app): u'Invalid email address.'] ## mismatching passwords - util.clear_test_template_context() + template.clear_test_template_context() test_app.post( '/auth/register/', { 'password': 'herpderp', 'confirm_password': 'derpherp'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] form = context['register_form'] assert form.password.errors == [ @@ -142,7 +142,7 @@ def test_register_views(test_app): # Successful register # ------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/register/', { 'username': 'happygirl', @@ -155,7 +155,7 @@ def test_register_views(test_app): assert_equal( urlparse.urlsplit(response.location)[2], '/u/happygirl/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/user_pages/user.html') ## Make sure user is in place @@ -166,15 +166,15 @@ def test_register_views(test_app): assert new_user['email_verified'] == False ## Make sure user is logged in - request = util.TEMPLATE_TEST_CONTEXT[ + request = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html']['request'] assert request.session['user_id'] == unicode(new_user['_id']) ## Make sure we get email confirmation, and try verifying - assert len(util.EMAIL_TEST_INBOX) == 1 - message = util.EMAIL_TEST_INBOX.pop() + assert len(template.EMAIL_TEST_INBOX) == 1 + message = template.EMAIL_TEST_INBOX.pop() assert message['To'] == 'happygrrl@example.org' - email_context = util.TEMPLATE_TEST_CONTEXT[ + email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/verification_email.txt'] assert email_context['verification_url'] in message.get_payload(decode=True) @@ -190,12 +190,12 @@ def test_register_views(test_app): new_user['verification_key']] ## Try verifying with bs verification key, shouldn't work - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.get( "/auth/verify_email/?userid=%s&token=total_bs" % unicode( new_user['_id'])) response.follow() - context = util.TEMPLATE_TEST_CONTEXT[ + context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html'] # assert context['verification_successful'] == True # TODO: Would be good to test messages here when we can do so... @@ -206,10 +206,10 @@ def test_register_views(test_app): assert new_user['email_verified'] == False ## Verify the email activation works - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.get("%s?%s" % (path, get_params)) response.follow() - context = util.TEMPLATE_TEST_CONTEXT[ + context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html'] # assert context['verification_successful'] == True # TODO: Would be good to test messages here when we can do so... @@ -222,7 +222,7 @@ def test_register_views(test_app): # Uniqueness checks # ----------------- ## We shouldn't be able to register with that user twice - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/register/', { 'username': 'happygirl', @@ -230,7 +230,7 @@ def test_register_views(test_app): 'confirm_password': 'iamsohappy2', 'email': 'happygrrl2@example.org'}) - context = util.TEMPLATE_TEST_CONTEXT[ + context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/register.html'] form = context['register_form'] assert form.username.errors == [ @@ -240,7 +240,7 @@ def test_register_views(test_app): ### Oops, forgot the password # ------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/forgot_password/', {'username': 'happygirl'}) @@ -250,14 +250,14 @@ def test_register_views(test_app): assert_equal( urlparse.urlsplit(response.location)[2], '/auth/forgot_password/email_sent/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/auth/fp_email_sent.html') ## Make sure link to change password is sent by email - assert len(util.EMAIL_TEST_INBOX) == 1 - message = util.EMAIL_TEST_INBOX.pop() + assert len(template.EMAIL_TEST_INBOX) == 1 + message = template.EMAIL_TEST_INBOX.pop() assert message['To'] == 'happygrrl@example.org' - email_context = util.TEMPLATE_TEST_CONTEXT[ + email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/fp_verification_email.txt'] #TODO - change the name of verification_url to something forgot-password-ish assert email_context['verification_url'] in message.get_payload(decode=True) @@ -277,14 +277,14 @@ def test_register_views(test_app): assert (new_user['fp_token_expire'] - datetime.datetime.now()).days == 9 ## Try using a bs password-changing verification key, shouldn't work - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.get( "/auth/forgot_password/verify/?userid=%s&token=total_bs" % unicode( new_user['_id']), status=400) assert response.status == '400 Bad Request' ## Try using an expired token to change password, shouldn't work - util.clear_test_template_context() + template.clear_test_template_context() real_token_expiration = new_user['fp_token_expire'] new_user['fp_token_expire'] = datetime.datetime.now() new_user.save() @@ -294,12 +294,12 @@ def test_register_views(test_app): new_user.save() ## Verify step 1 of password-change works -- can see form to change password - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.get("%s?%s" % (path, get_params)) - assert util.TEMPLATE_TEST_CONTEXT.has_key('mediagoblin/auth/change_fp.html') + assert template.TEMPLATE_TEST_CONTEXT.has_key('mediagoblin/auth/change_fp.html') ## Verify step 2.1 of password-change works -- report success to user - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/forgot_password/verify/', { 'userid': parsed_get_params['userid'], @@ -307,11 +307,11 @@ def test_register_views(test_app): 'confirm_password': 'iamveryveryhappy', 'token': parsed_get_params['token']}) response.follow() - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/auth/fp_changed_success.html') ## Verify step 2.2 of password-change works -- login w/ new password success - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'happygirl', @@ -322,7 +322,7 @@ def test_register_views(test_app): assert_equal( urlparse.urlsplit(response.location)[2], '/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/root.html') @@ -341,61 +341,61 @@ def test_authentication_views(test_app): # Get login # --------- test_app.get('/auth/login/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/auth/login.html') # Failed login - blank form # ------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post('/auth/login/') - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] form = context['login_form'] assert form.username.errors == [u'This field is required.'] assert form.password.errors == [u'This field is required.'] # Failed login - blank user # ------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'password': u'toast'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] form = context['login_form'] assert form.username.errors == [u'This field is required.'] # Failed login - blank password # ----------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'chris'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] form = context['login_form'] assert form.password.errors == [u'This field is required.'] # Failed login - bad user # ----------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'steve', 'password': 'toast'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] assert context['login_failed'] # Failed login - bad password # --------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'chris', 'password': 'jam'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/login.html'] assert context['login_failed'] # Successful login # ---------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'chris', @@ -406,17 +406,17 @@ def test_authentication_views(test_app): assert_equal( urlparse.urlsplit(response.location)[2], '/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/root.html') # Make sure user is in the session - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session assert session['user_id'] == unicode(test_user['_id']) # Successful logout # ----------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.get('/auth/logout/') # Should be redirected to index page @@ -424,17 +424,17 @@ def test_authentication_views(test_app): assert_equal( urlparse.urlsplit(response.location)[2], '/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/root.html') # Make sure the user is not in the session - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session assert session.has_key('user_id') == False # User is redirected to custom URL if POST['next'] is set # ------------------------------------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = test_app.post( '/auth/login/', { 'username': u'chris', diff --git a/mediagoblin/tests/test_messages.py b/mediagoblin/tests/test_messages.py index 9c57a151..2635f4d7 100644 --- a/mediagoblin/tests/test_messages.py +++ b/mediagoblin/tests/test_messages.py @@ -16,7 +16,7 @@ from mediagoblin.messages import fetch_messages, add_message from mediagoblin.tests.tools import setup_fresh_app -from mediagoblin import util +from mediagoblin.tools import template @setup_fresh_app @@ -28,7 +28,7 @@ def test_messages(test_app): """ # Aquire a request object test_app.get('/') - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] request = context['request'] # The message queue should be empty diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index 007c0348..1c657e6c 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -22,7 +22,7 @@ from nose.tools import assert_equal, assert_true, assert_false from mediagoblin.auth import lib as auth_lib from mediagoblin.tests.tools import setup_fresh_app, get_test_app from mediagoblin import mg_globals -from mediagoblin import util +from mediagoblin.tools import template, common GOOD_JPG = pkg_resources.resource_filename( 'mediagoblin.tests', 'test_submission/good.jpg') @@ -63,20 +63,20 @@ class TestSubmission: def test_missing_fields(self): # Test blank form # --------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', {}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] form = context['submit_form'] assert form.file.errors == [u'You must provide a file.'] # Test blank file # --------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'test title'}) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] form = context['submit_form'] assert form.file.errors == [u'You must provide a file.'] @@ -84,7 +84,7 @@ class TestSubmission: def test_normal_uploads(self): # Test JPG # -------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Normal upload 1' @@ -96,12 +96,12 @@ class TestSubmission: assert_equal( urlparse.urlsplit(response.location)[2], '/u/chris/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/user_pages/user.html') # Test PNG # -------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Normal upload 2' @@ -112,13 +112,13 @@ class TestSubmission: assert_equal( urlparse.urlsplit(response.location)[2], '/u/chris/') - assert util.TEMPLATE_TEST_CONTEXT.has_key( + assert template.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/user_pages/user.html') def test_tags(self): # Good tag string # -------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Balanced Goblin', @@ -128,7 +128,7 @@ class TestSubmission: # New media entry with correct tags should be created response.follow() - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/user_pages/user.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/user_pages/user.html'] request = context['request'] media = request.db.MediaEntry.find({'title': 'Balanced Goblin'})[0] assert_equal(media['tags'], @@ -137,7 +137,7 @@ class TestSubmission: # Test tags that are too long # --------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Balanced Goblin', @@ -146,14 +146,14 @@ class TestSubmission: 'file', GOOD_JPG)]) # Too long error should be raised - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] form = context['submit_form'] assert form.tags.errors == [ u'Tags must be shorter than 50 characters. Tags that are too long'\ ': ffffffffffffffffffffffffffuuuuuuuuuuuuuuuuuuuuuuuuuu'] def test_delete(self): - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Balanced Goblin', @@ -163,7 +163,7 @@ class TestSubmission: # Post image response.follow() - request = util.TEMPLATE_TEST_CONTEXT[ + request = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html']['request'] media = request.db.MediaEntry.find({'title': 'Balanced Goblin'})[0] @@ -183,7 +183,7 @@ class TestSubmission: response.follow() - request = util.TEMPLATE_TEST_CONTEXT[ + request = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html']['request'] media = request.db.MediaEntry.find({'title': 'Balanced Goblin'})[0] @@ -202,7 +202,7 @@ class TestSubmission: response.follow() - request = util.TEMPLATE_TEST_CONTEXT[ + request = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user.html']['request'] # Does media entry still exist? @@ -213,14 +213,14 @@ class TestSubmission: def test_malicious_uploads(self): # Test non-suppoerted file with non-supported extension # ----------------------------------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Malicious Upload 1' }, upload_files=[( 'file', EVIL_FILE)]) - context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] + context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/submit/start.html'] form = context['submit_form'] assert form.file.errors == ['The file doesn\'t seem to be an image!'] @@ -230,7 +230,7 @@ class TestSubmission: # Test non-supported file with .jpg extension # ------------------------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Malicious Upload 2' @@ -250,7 +250,7 @@ class TestSubmission: # Test non-supported file with .png extension # ------------------------------------------- - util.clear_test_template_context() + template.clear_test_template_context() response = self.test_app.post( '/submit/', { 'title': 'Malicious Upload 3' diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index c2a3a67f..cdc62b7d 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -17,7 +17,7 @@ import email from mediagoblin import util - +from mediagoblin.tools import url, translate util._activate_testing() @@ -71,38 +71,38 @@ I hope you like unit tests JUST AS MUCH AS I DO!""" I hope you like unit tests JUST AS MUCH AS I DO!""" def test_slugify(): - assert util.slugify('a walk in the park') == 'a-walk-in-the-park' - assert util.slugify('A Walk in the Park') == 'a-walk-in-the-park' - assert util.slugify('a walk in the park') == 'a-walk-in-the-park' - assert util.slugify('a walk in-the-park') == 'a-walk-in-the-park' - assert util.slugify('a w@lk in the park?') == 'a-w-lk-in-the-park' - assert util.slugify(u'a walk in the par\u0107') == 'a-walk-in-the-parc' - assert util.slugify(u'\u00E0\u0042\u00E7\u010F\u00EB\u0066') == 'abcdef' + assert url.slugify('a walk in the park') == 'a-walk-in-the-park' + assert url.slugify('A Walk in the Park') == 'a-walk-in-the-park' + assert url.slugify('a walk in the park') == 'a-walk-in-the-park' + assert url.slugify('a walk in-the-park') == 'a-walk-in-the-park' + assert url.slugify('a w@lk in the park?') == 'a-w-lk-in-the-park' + assert url.slugify(u'a walk in the par\u0107') == 'a-walk-in-the-parc' + assert url.slugify(u'\u00E0\u0042\u00E7\u010F\u00EB\u0066') == 'abcdef' def test_locale_to_lower_upper(): """ Test cc.i18n.util.locale_to_lower_upper() """ - assert util.locale_to_lower_upper('en') == 'en' - assert util.locale_to_lower_upper('en_US') == 'en_US' - assert util.locale_to_lower_upper('en-us') == 'en_US' + assert translate.locale_to_lower_upper('en') == 'en' + assert translate.locale_to_lower_upper('en_US') == 'en_US' + assert translate.locale_to_lower_upper('en-us') == 'en_US' # crazy renditions. Useful? - assert util.locale_to_lower_upper('en-US') == 'en_US' - assert util.locale_to_lower_upper('en_us') == 'en_US' + assert translate.locale_to_lower_upper('en-US') == 'en_US' + assert translate.locale_to_lower_upper('en_us') == 'en_US' def test_locale_to_lower_lower(): """ Test cc.i18n.util.locale_to_lower_lower() """ - assert util.locale_to_lower_lower('en') == 'en' - assert util.locale_to_lower_lower('en_US') == 'en-us' - assert util.locale_to_lower_lower('en-us') == 'en-us' + assert translate.locale_to_lower_lower('en') == 'en' + assert translate.locale_to_lower_lower('en_US') == 'en-us' + assert translate.locale_to_lower_lower('en-us') == 'en-us' # crazy renditions. Useful? - assert util.locale_to_lower_lower('en-US') == 'en-us' - assert util.locale_to_lower_lower('en_us') == 'en-us' + assert translate.locale_to_lower_lower('en-US') == 'en-us' + assert translate.locale_to_lower_lower('en_us') == 'en-us' def test_html_cleaner(): diff --git a/mediagoblin/tools/__init__.py b/mediagoblin/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mediagoblin/tools/common.py b/mediagoblin/tools/common.py new file mode 100644 index 00000000..dccceccb --- /dev/null +++ b/mediagoblin/tools/common.py @@ -0,0 +1,18 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +global TESTS_ENABLED +TESTS_ENABLED = False diff --git a/mediagoblin/tools/template.py b/mediagoblin/tools/template.py new file mode 100644 index 00000000..c346c33d --- /dev/null +++ b/mediagoblin/tools/template.py @@ -0,0 +1,114 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from math import ceil +import jinja2 +from babel.localedata import exists +from babel.support import LazyProxy +from mediagoblin import mg_globals +from mediagoblin import messages +from mediagoblin.tools import common +from mediagoblin.tools.translate import setup_gettext + +SETUP_JINJA_ENVS = {} + +def get_jinja_env(template_loader, locale): + """ + Set up the Jinja environment, + + (In the future we may have another system for providing theming; + for now this is good enough.) + """ + setup_gettext(locale) + + # If we have a jinja environment set up with this locale, just + # return that one. + if SETUP_JINJA_ENVS.has_key(locale): + return SETUP_JINJA_ENVS[locale] + + template_env = jinja2.Environment( + loader=template_loader, autoescape=True, + extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape']) + + template_env.install_gettext_callables( + mg_globals.translations.ugettext, + mg_globals.translations.ungettext) + + # All templates will know how to ... + # ... fetch all waiting messages and remove them from the queue + # ... construct a grid of thumbnails or other media + template_env.globals['fetch_messages'] = messages.fetch_messages + template_env.globals['gridify_list'] = gridify_list + template_env.globals['gridify_cursor'] = gridify_cursor + + if exists(locale): + SETUP_JINJA_ENVS[locale] = template_env + + return template_env + +# We'll store context information here when doing unit tests +TEMPLATE_TEST_CONTEXT = {} + + +def render_template(request, template_path, context): + """ + Render a template with context. + + Always inserts the request into the context, so you don't have to. + Also stores the context if we're doing unit tests. Helpful! + """ + template = request.template_env.get_template( + template_path) + context['request'] = request + rendered = template.render(context) + + if common.TESTS_ENABLED: + TEMPLATE_TEST_CONTEXT[template_path] = context + + return rendered + + +def clear_test_template_context(): + global TEMPLATE_TEST_CONTEXT + TEMPLATE_TEST_CONTEXT = {} + +def gridify_list(this_list, num_cols=5): + """ + Generates a list of lists where each sub-list's length depends on + the number of columns in the list + """ + grid = [] + + # Figure out how many rows we should have + num_rows = int(ceil(float(len(this_list)) / num_cols)) + + for row_num in range(num_rows): + slice_min = row_num * num_cols + slice_max = (row_num + 1) * num_cols + + row = this_list[slice_min:slice_max] + + grid.append(row) + + return grid + + +def gridify_cursor(this_cursor, num_cols=5): + """ + Generates a list of lists where each sub-list's length depends on + the number of columns in the list + """ + return gridify_list(list(this_cursor), num_cols) diff --git a/mediagoblin/tools/translate.py b/mediagoblin/tools/translate.py new file mode 100644 index 00000000..2c2a710d --- /dev/null +++ b/mediagoblin/tools/translate.py @@ -0,0 +1,167 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import gettext +import pkg_resources +from babel.localedata import exists +from babel.support import LazyProxy + +from mediagoblin import mg_globals + +################### +# Translation tools +################### + + +TRANSLATIONS_PATH = pkg_resources.resource_filename( + 'mediagoblin', 'i18n') + + +def locale_to_lower_upper(locale): + """ + Take a locale, regardless of style, and format it like "en-us" + """ + if '-' in locale: + lang, country = locale.split('-', 1) + return '%s_%s' % (lang.lower(), country.upper()) + elif '_' in locale: + lang, country = locale.split('_', 1) + return '%s_%s' % (lang.lower(), country.upper()) + else: + return locale.lower() + + +def locale_to_lower_lower(locale): + """ + Take a locale, regardless of style, and format it like "en_US" + """ + if '_' in locale: + lang, country = locale.split('_', 1) + return '%s-%s' % (lang.lower(), country.lower()) + else: + return locale.lower() + + +def get_locale_from_request(request): + """ + Figure out what target language is most appropriate based on the + request + """ + request_form = request.GET or request.POST + + if request_form.has_key('lang'): + return locale_to_lower_upper(request_form['lang']) + + accept_lang_matches = request.accept_language.best_matches() + + # Your routing can explicitly specify a target language + matchdict = request.matchdict or {} + + if matchdict.has_key('locale'): + target_lang = matchdict['locale'] + elif request.session.has_key('target_lang'): + target_lang = request.session['target_lang'] + # Pull the first acceptable language + elif accept_lang_matches: + target_lang = accept_lang_matches[0] + # Fall back to English + else: + target_lang = 'en' + + return locale_to_lower_upper(target_lang) + +SETUP_GETTEXTS = {} + +def setup_gettext(locale): + """ + Setup the gettext instance based on this locale + """ + # Later on when we have plugins we may want to enable the + # multi-translations system they have so we can handle plugin + # translations too + + # TODO: fallback nicely on translations from pt_PT to pt if not + # available, etc. + if SETUP_GETTEXTS.has_key(locale): + this_gettext = SETUP_GETTEXTS[locale] + else: + this_gettext = gettext.translation( + 'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True) + if exists(locale): + SETUP_GETTEXTS[locale] = this_gettext + + mg_globals.setup_globals( + translations=this_gettext) + + +# Force en to be setup before anything else so that +# mg_globals.translations is never None +setup_gettext('en') + + +def pass_to_ugettext(*args, **kwargs): + """ + Pass a translation on to the appropriate ugettext method. + + The reason we can't have a global ugettext method is because + mg_globals gets swapped out by the application per-request. + """ + return mg_globals.translations.ugettext( + *args, **kwargs) + + +def lazy_pass_to_ugettext(*args, **kwargs): + """ + Lazily pass to ugettext. + + This is useful if you have to define a translation on a module + level but you need it to not translate until the time that it's + used as a string. + """ + return LazyProxy(pass_to_ugettext, *args, **kwargs) + + +def pass_to_ngettext(*args, **kwargs): + """ + Pass a translation on to the appropriate ngettext method. + + The reason we can't have a global ngettext method is because + mg_globals gets swapped out by the application per-request. + """ + return mg_globals.translations.ngettext( + *args, **kwargs) + + +def lazy_pass_to_ngettext(*args, **kwargs): + """ + Lazily pass to ngettext. + + This is useful if you have to define a translation on a module + level but you need it to not translate until the time that it's + used as a string. + """ + return LazyProxy(pass_to_ngettext, *args, **kwargs) + + +def fake_ugettext_passthrough(string): + """ + Fake a ugettext call for extraction's sake ;) + + In wtforms there's a separate way to define a method to translate + things... so we just need to mark up the text so that it can be + extracted, not so that it's actually run through gettext. + """ + return string diff --git a/mediagoblin/tools/url.py b/mediagoblin/tools/url.py new file mode 100644 index 00000000..458ef2c8 --- /dev/null +++ b/mediagoblin/tools/url.py @@ -0,0 +1,31 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re +import translitcodec + +_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') + +def slugify(text, delim=u'-'): + """ + Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/ + """ + result = [] + for word in _punct_re.split(text.lower()): + word = word.encode('translit/long') + if word: + result.append(word) + return unicode(delim.join(result)) diff --git a/mediagoblin/user_pages/forms.py b/mediagoblin/user_pages/forms.py index 57061d34..301f1f0a 100644 --- a/mediagoblin/user_pages/forms.py +++ b/mediagoblin/user_pages/forms.py @@ -16,7 +16,7 @@ import wtforms -from mediagoblin.util import fake_ugettext_passthrough as _ +from mediagoblin.tools.translate import fake_ugettext_passthrough as _ class MediaCommentForm(wtforms.Form): diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 6a82d718..40c7ffce 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -21,7 +21,7 @@ from mediagoblin.db.util import DESCENDING, ObjectId from mediagoblin.util import ( Pagination, render_to_response, redirect, cleaned_markdown_conversion, render_404, delete_media_files) -from mediagoblin.util import pass_to_ugettext as _ +from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.user_pages import forms as user_forms from mediagoblin.decorators import (uses_pagination, get_user_media_entry, diff --git a/mediagoblin/util.py b/mediagoblin/util.py index 7ff3ec7f..35755ccf 100644 --- a/mediagoblin/util.py +++ b/mediagoblin/util.py @@ -17,41 +17,42 @@ from __future__ import division from email.MIMEText import MIMEText -import gettext -import pkg_resources +#import gettext +#import pkg_resources import smtplib import sys -import re +#import re +#import translitcodec import urllib from math import ceil, floor import copy import wtforms -from babel.localedata import exists -from babel.support import LazyProxy -import jinja2 -import translitcodec +#from babel.localedata import exists +#from babel.support import LazyProxy +#import jinja2 from webob import Response, exc from lxml.html.clean import Cleaner import markdown from wtforms.form import Form from mediagoblin import mg_globals -from mediagoblin import messages +#from mediagoblin import messages from mediagoblin.db.util import ObjectId +from mediagoblin.tools import url +from mediagoblin.tools import common +from mediagoblin.tools.template import TEMPLATE_TEST_CONTEXT, render_template from itertools import izip, count DISPLAY_IMAGE_FETCHING_ORDER = [u'medium', u'original', u'thumb'] -TESTS_ENABLED = False def _activate_testing(): """ Call this to activate testing in util.py """ - global TESTS_ENABLED - TESTS_ENABLED = True + common.TESTS_ENABLED = True def clear_test_buckets(): """ @@ -73,64 +74,64 @@ def clear_test_buckets(): clear_test_template_context() -SETUP_JINJA_ENVS = {} +# SETUP_JINJA_ENVS = {} -def get_jinja_env(template_loader, locale): - """ - Set up the Jinja environment, +# def get_jinja_env(template_loader, locale): +# """ +# Set up the Jinja environment, - (In the future we may have another system for providing theming; - for now this is good enough.) - """ - setup_gettext(locale) +# (In the future we may have another system for providing theming; +# for now this is good enough.) +# """ +# setup_gettext(locale) - # If we have a jinja environment set up with this locale, just - # return that one. - if SETUP_JINJA_ENVS.has_key(locale): - return SETUP_JINJA_ENVS[locale] +# # If we have a jinja environment set up with this locale, just +# # return that one. +# if SETUP_JINJA_ENVS.has_key(locale): +# return SETUP_JINJA_ENVS[locale] - template_env = jinja2.Environment( - loader=template_loader, autoescape=True, - extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape']) +# template_env = jinja2.Environment( +# loader=template_loader, autoescape=True, +# extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape']) - template_env.install_gettext_callables( - mg_globals.translations.ugettext, - mg_globals.translations.ungettext) +# template_env.install_gettext_callables( +# mg_globals.translations.ugettext, +# mg_globals.translations.ungettext) - # All templates will know how to ... - # ... fetch all waiting messages and remove them from the queue - # ... construct a grid of thumbnails or other media - template_env.globals['fetch_messages'] = messages.fetch_messages - template_env.globals['gridify_list'] = gridify_list - template_env.globals['gridify_cursor'] = gridify_cursor +# # All templates will know how to ... +# # ... fetch all waiting messages and remove them from the queue +# # ... construct a grid of thumbnails or other media +# template_env.globals['fetch_messages'] = messages.fetch_messages +# template_env.globals['gridify_list'] = gridify_list +# template_env.globals['gridify_cursor'] = gridify_cursor - if exists(locale): - SETUP_JINJA_ENVS[locale] = template_env +# if exists(locale): +# SETUP_JINJA_ENVS[locale] = template_env - return template_env +# return template_env -# We'll store context information here when doing unit tests -TEMPLATE_TEST_CONTEXT = {} +# # We'll store context information here when doing unit tests +# TEMPLATE_TEST_CONTEXT = {} -def render_template(request, template_path, context): - """ - Render a template with context. +# def render_template(request, template_path, context): +# """ +# Render a template with context. - Always inserts the request into the context, so you don't have to. - Also stores the context if we're doing unit tests. Helpful! - """ - template = request.template_env.get_template( - template_path) - context['request'] = request - rendered = template.render(context) +# Always inserts the request into the context, so you don't have to. +# Also stores the context if we're doing unit tests. Helpful! +# """ +# template = request.template_env.get_template( +# template_path) +# context['request'] = request +# rendered = template.render(context) - if TESTS_ENABLED: - TEMPLATE_TEST_CONTEXT[template_path] = context +# if TESTS_ENABLED: +# TEMPLATE_TEST_CONTEXT[template_path] = context - return rendered +# return rendered def clear_test_template_context(): @@ -195,18 +196,18 @@ def import_component(import_string): func = getattr(module, func_name) return func -_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') +# _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') -def slugify(text, delim=u'-'): - """ - Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/ - """ - result = [] - for word in _punct_re.split(text.lower()): - word = word.encode('translit/long') - if word: - result.append(word) - return unicode(delim.join(result)) +# def slugify(text, delim=u'-'): +# """ +# Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/ +# """ +# result = [] +# for word in _punct_re.split(text.lower()): +# word = word.encode('translit/long') +# if word: +# result.append(word) +# return unicode(delim.join(result)) ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ### Special email test stuff begins HERE @@ -274,7 +275,7 @@ def send_email(from_addr, to_addrs, subject, message_body): - subject: subject of the email - message_body: email body text """ - if TESTS_ENABLED or mg_globals.app_config['email_debug_mode']: + if common.TESTS_ENABLED or mg_globals.app_config['email_debug_mode']: mhost = FakeMhost() elif not mg_globals.app_config['email_debug_mode']: mhost = smtplib.SMTP( @@ -296,7 +297,7 @@ def send_email(from_addr, to_addrs, subject, message_body): message['From'] = from_addr message['To'] = ', '.join(to_addrs) - if TESTS_ENABLED: + if common.TESTS_ENABLED: EMAIL_TEST_INBOX.append(message) if mg_globals.app_config['email_debug_mode']: @@ -310,67 +311,67 @@ def send_email(from_addr, to_addrs, subject, message_body): return mhost.sendmail(from_addr, to_addrs, message.as_string()) -################### -# Translation tools -################### +# ################### +# # Translation tools +# ################### -TRANSLATIONS_PATH = pkg_resources.resource_filename( - 'mediagoblin', 'i18n') +# TRANSLATIONS_PATH = pkg_resources.resource_filename( +# 'mediagoblin', 'i18n') -def locale_to_lower_upper(locale): - """ - Take a locale, regardless of style, and format it like "en-us" - """ - if '-' in locale: - lang, country = locale.split('-', 1) - return '%s_%s' % (lang.lower(), country.upper()) - elif '_' in locale: - lang, country = locale.split('_', 1) - return '%s_%s' % (lang.lower(), country.upper()) - else: - return locale.lower() +# def locale_to_lower_upper(locale): +# """ +# Take a locale, regardless of style, and format it like "en-us" +# """ +# if '-' in locale: +# lang, country = locale.split('-', 1) +# return '%s_%s' % (lang.lower(), country.upper()) +# elif '_' in locale: +# lang, country = locale.split('_', 1) +# return '%s_%s' % (lang.lower(), country.upper()) +# else: +# return locale.lower() -def locale_to_lower_lower(locale): - """ - Take a locale, regardless of style, and format it like "en_US" - """ - if '_' in locale: - lang, country = locale.split('_', 1) - return '%s-%s' % (lang.lower(), country.lower()) - else: - return locale.lower() +# def locale_to_lower_lower(locale): +# """ +# Take a locale, regardless of style, and format it like "en_US" +# """ +# if '_' in locale: +# lang, country = locale.split('_', 1) +# return '%s-%s' % (lang.lower(), country.lower()) +# else: +# return locale.lower() -def get_locale_from_request(request): - """ - Figure out what target language is most appropriate based on the - request - """ - request_form = request.GET or request.POST +# def get_locale_from_request(request): +# """ +# Figure out what target language is most appropriate based on the +# request +# """ +# request_form = request.GET or request.POST - if request_form.has_key('lang'): - return locale_to_lower_upper(request_form['lang']) +# if request_form.has_key('lang'): +# return locale_to_lower_upper(request_form['lang']) - accept_lang_matches = request.accept_language.best_matches() +# accept_lang_matches = request.accept_language.best_matches() - # Your routing can explicitly specify a target language - matchdict = request.matchdict or {} +# # Your routing can explicitly specify a target language +# matchdict = request.matchdict or {} - if matchdict.has_key('locale'): - target_lang = matchdict['locale'] - elif request.session.has_key('target_lang'): - target_lang = request.session['target_lang'] - # Pull the first acceptable language - elif accept_lang_matches: - target_lang = accept_lang_matches[0] - # Fall back to English - else: - target_lang = 'en' +# if matchdict.has_key('locale'): +# target_lang = matchdict['locale'] +# elif request.session.has_key('target_lang'): +# target_lang = request.session['target_lang'] +# # Pull the first acceptable language +# elif accept_lang_matches: +# target_lang = accept_lang_matches[0] +# # Fall back to English +# else: +# target_lang = 'en' - return locale_to_lower_upper(target_lang) +# return locale_to_lower_upper(target_lang) # A super strict version of the lxml.html cleaner class @@ -424,7 +425,7 @@ def convert_to_tag_list_of_dicts(tag_string): if tag.strip() and tag.strip() not in [t['name'] for t in taglist]: taglist.append({'name': tag.strip(), - 'slug': slugify(tag.strip())}) + 'slug': url.slugify(tag.strip())}) return taglist @@ -472,88 +473,88 @@ def cleaned_markdown_conversion(text): return clean_html(MARKDOWN_INSTANCE.convert(text)) -SETUP_GETTEXTS = {} +# SETUP_GETTEXTS = {} -def setup_gettext(locale): - """ - Setup the gettext instance based on this locale - """ - # Later on when we have plugins we may want to enable the - # multi-translations system they have so we can handle plugin - # translations too +# def setup_gettext(locale): +# """ +# Setup the gettext instance based on this locale +# """ +# # Later on when we have plugins we may want to enable the +# # multi-translations system they have so we can handle plugin +# # translations too - # TODO: fallback nicely on translations from pt_PT to pt if not - # available, etc. - if SETUP_GETTEXTS.has_key(locale): - this_gettext = SETUP_GETTEXTS[locale] - else: - this_gettext = gettext.translation( - 'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True) - if exists(locale): - SETUP_GETTEXTS[locale] = this_gettext +# # TODO: fallback nicely on translations from pt_PT to pt if not +# # available, etc. +# if SETUP_GETTEXTS.has_key(locale): +# this_gettext = SETUP_GETTEXTS[locale] +# else: +# this_gettext = gettext.translation( +# 'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True) +# if exists(locale): +# SETUP_GETTEXTS[locale] = this_gettext - mg_globals.setup_globals( - translations=this_gettext) +# mg_globals.setup_globals( +# translations=this_gettext) -# Force en to be setup before anything else so that -# mg_globals.translations is never None -setup_gettext('en') +# # Force en to be setup before anything else so that +# # mg_globals.translations is never None +# setup_gettext('en') -def pass_to_ugettext(*args, **kwargs): - """ - Pass a translation on to the appropriate ugettext method. +# def pass_to_ugettext(*args, **kwargs): +# """ +# Pass a translation on to the appropriate ugettext method. - The reason we can't have a global ugettext method is because - mg_globals gets swapped out by the application per-request. - """ - return mg_globals.translations.ugettext( - *args, **kwargs) +# The reason we can't have a global ugettext method is because +# mg_globals gets swapped out by the application per-request. +# """ +# return mg_globals.translations.ugettext( +# *args, **kwargs) -def lazy_pass_to_ugettext(*args, **kwargs): - """ - Lazily pass to ugettext. +# def lazy_pass_to_ugettext(*args, **kwargs): +# """ +# Lazily pass to ugettext. - This is useful if you have to define a translation on a module - level but you need it to not translate until the time that it's - used as a string. - """ - return LazyProxy(pass_to_ugettext, *args, **kwargs) +# This is useful if you have to define a translation on a module +# level but you need it to not translate until the time that it's +# used as a string. +# """ +# return LazyProxy(pass_to_ugettext, *args, **kwargs) -def pass_to_ngettext(*args, **kwargs): - """ - Pass a translation on to the appropriate ngettext method. +# def pass_to_ngettext(*args, **kwargs): +# """ +# Pass a translation on to the appropriate ngettext method. - The reason we can't have a global ngettext method is because - mg_globals gets swapped out by the application per-request. - """ - return mg_globals.translations.ngettext( - *args, **kwargs) +# The reason we can't have a global ngettext method is because +# mg_globals gets swapped out by the application per-request. +# """ +# return mg_globals.translations.ngettext( +# *args, **kwargs) -def lazy_pass_to_ngettext(*args, **kwargs): - """ - Lazily pass to ngettext. +# def lazy_pass_to_ngettext(*args, **kwargs): +# """ +# Lazily pass to ngettext. - This is useful if you have to define a translation on a module - level but you need it to not translate until the time that it's - used as a string. - """ - return LazyProxy(pass_to_ngettext, *args, **kwargs) +# This is useful if you have to define a translation on a module +# level but you need it to not translate until the time that it's +# used as a string. +# """ +# return LazyProxy(pass_to_ngettext, *args, **kwargs) -def fake_ugettext_passthrough(string): - """ - Fake a ugettext call for extraction's sake ;) +# def fake_ugettext_passthrough(string): +# """ +# Fake a ugettext call for extraction's sake ;) - In wtforms there's a separate way to define a method to translate - things... so we just need to mark up the text so that it can be - extracted, not so that it's actually run through gettext. - """ - return string +# In wtforms there's a separate way to define a method to translate +# things... so we just need to mark up the text so that it can be +# extracted, not so that it's actually run through gettext. +# """ +# return string PAGINATION_DEFAULT_PER_PAGE = 30 @@ -646,33 +647,33 @@ class Pagination(object): request.path_info, request.GET, page_no) -def gridify_list(this_list, num_cols=5): - """ - Generates a list of lists where each sub-list's length depends on - the number of columns in the list - """ - grid = [] +# def gridify_list(this_list, num_cols=5): +# """ +# Generates a list of lists where each sub-list's length depends on +# the number of columns in the list +# """ +# grid = [] - # Figure out how many rows we should have - num_rows = int(ceil(float(len(this_list)) / num_cols)) +# # Figure out how many rows we should have +# num_rows = int(ceil(float(len(this_list)) / num_cols)) - for row_num in range(num_rows): - slice_min = row_num * num_cols - slice_max = (row_num + 1) * num_cols +# for row_num in range(num_rows): +# slice_min = row_num * num_cols +# slice_max = (row_num + 1) * num_cols - row = this_list[slice_min:slice_max] +# row = this_list[slice_min:slice_max] - grid.append(row) +# grid.append(row) - return grid +# return grid -def gridify_cursor(this_cursor, num_cols=5): - """ - Generates a list of lists where each sub-list's length depends on - the number of columns in the list - """ - return gridify_list(list(this_cursor), num_cols) +# def gridify_cursor(this_cursor, num_cols=5): +# """ +# Generates a list of lists where each sub-list's length depends on +# the number of columns in the list +# """ +# return gridify_list(list(this_cursor), num_cols) def render_404(request): From 5d2abe45b2bae9111d4f1bda645b53414d2b240d Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sat, 1 Oct 2011 12:48:43 -0700 Subject: [PATCH 06/13] PEP8-ification. --- mediagoblin/middleware/csrf.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mediagoblin/middleware/csrf.py b/mediagoblin/middleware/csrf.py index a372d0b5..68ece6d3 100644 --- a/mediagoblin/middleware/csrf.py +++ b/mediagoblin/middleware/csrf.py @@ -34,17 +34,19 @@ class CsrfForm(Form): """Simple form to handle rendering a CSRF token and confirming it is included in the POST.""" - csrf_token = HiddenField("", + csrf_token = HiddenField("", [validators.Required()]) + def render_csrf_form_token(request): """Render the CSRF token in a format suitable for inclusion in a form.""" - form = CsrfForm(csrf_token = request.environ['CSRF_TOKEN']) + form = CsrfForm(csrf_token=request.environ['CSRF_TOKEN']) return form.csrf_token + class CsrfMiddleware(object): """CSRF Protection Middleware @@ -87,7 +89,8 @@ class CsrfMiddleware(object): response.set_cookie( mg_globals.app_config['csrf_cookie_name'], request.environ['CSRF_TOKEN'], - max_age=60*60*24*7*52, path='/', + max_age=60 * 60 * 24 * 7 * 52, + path='/', domain=mg_globals.app_config.get('csrf_cookie_domain', None), secure=(request.scheme.lower() == 'https'), httponly=True) @@ -98,10 +101,9 @@ class CsrfMiddleware(object): def _make_token(self, request): """Generate a new token to use for CSRF protection.""" - return hashlib.md5("%s%s" % - (randrange(0, self.MAX_CSRF_KEY), - mg_globals.app_config['secret_key']) - ).hexdigest() + return hashlib.md5("%s%s" % + (randrange(0, self.MAX_CSRF_KEY), + mg_globals.app_config['secret_key'])).hexdigest() def verify_tokens(self, request): """Verify that the CSRF Cookie exists and that it matches the @@ -109,7 +111,7 @@ class CsrfMiddleware(object): # confirm the cookie token was presented cookie_token = request.cookies.get( - mg_globals.app_config['csrf_cookie_name'], + mg_globals.app_config['csrf_cookie_name'], None) if cookie_token is None: @@ -128,4 +130,3 @@ class CsrfMiddleware(object): # either the tokens didn't match or the form token wasn't # present; either way, the request is denied return HTTPForbidden() - From 7e694e5fd858aeaea7eb7e9a9062b36d17a3b7f7 Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sat, 1 Oct 2011 13:13:14 -0700 Subject: [PATCH 07/13] #361: Don't test for CSRF token if we're running unit tests. --- mediagoblin/middleware/csrf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mediagoblin/middleware/csrf.py b/mediagoblin/middleware/csrf.py index 68ece6d3..d41bcd87 100644 --- a/mediagoblin/middleware/csrf.py +++ b/mediagoblin/middleware/csrf.py @@ -77,7 +77,10 @@ class CsrfMiddleware(object): # if this is a non-"safe" request (ie, one that could have # side effects), confirm that the CSRF tokens are present and # valid - if request.method not in self.SAFE_HTTP_METHODS: + if request.method not in self.SAFE_HTTP_METHODS \ + and ('gmg.verify_csrf' in request.environ or + 'paste.testing' not in request.environ): + return self.verify_tokens(request) def process_response(self, request, response): From 4f475d3024f689c1c461dc26bd679dfb514a46ef Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sat, 1 Oct 2011 14:21:02 -0700 Subject: [PATCH 08/13] #361 Unit tests for CSRF Middleware --- mediagoblin/tests/test_csrf_middleware.py | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 mediagoblin/tests/test_csrf_middleware.py diff --git a/mediagoblin/tests/test_csrf_middleware.py b/mediagoblin/tests/test_csrf_middleware.py new file mode 100644 index 00000000..cf03fe58 --- /dev/null +++ b/mediagoblin/tests/test_csrf_middleware.py @@ -0,0 +1,69 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import urlparse +import datetime + +from nose.tools import assert_equal + +from mediagoblin.tests.tools import setup_fresh_app +from mediagoblin import mg_globals + + +@setup_fresh_app +def test_csrf_cookie_set(test_app): + + # get login page + response = test_app.get('/auth/login/') + + # assert that the mediagoblin nonce cookie has been set + assert 'Set-Cookie' in response.headers + assert 'mediagoblin_nonce' in response.cookies_set + + # assert that we're also sending a vary header + assert response.headers.get('Vary', False) == 'Cookie' + + +@setup_fresh_app +def test_csrf_token_must_match(test_app): + + # construct a request with no cookie or form token + assert test_app.post('/auth/login/', + extra_environ={'gmg.verify_csrf': True}, + expect_errors=True).status_int == 403 + + # construct a request with a cookie, but no form token + assert test_app.post('/auth/login/', + headers={'Cookie': str('%s=foo; ' % + mg_globals.app_config['csrf_cookie_name'])}, + extra_environ={'gmg.verify_csrf': True}, + expect_errors=True).status_int == 403 + + # if both the cookie and form token are provided, they must match + assert test_app.post('/auth/login/', + {'csrf_token': 'blarf'}, + headers={'Cookie': str('%s=foo; ' % + mg_globals.app_config['csrf_cookie_name'])}, + extra_environ={'gmg.verify_csrf': True}, + expect_errors=True).\ + status_int == 403 + + assert test_app.post('/auth/login/', + {'csrf_token': 'foo'}, + headers={'Cookie': str('%s=foo; ' % + mg_globals.app_config['csrf_cookie_name'])}, + extra_environ={'gmg.verify_csrf': True}).\ + status_int == 200 From 9202e5a1e15183b134fa15c4e1290dea8ed2acbe Mon Sep 17 00:00:00 2001 From: Nathan Yergler Date: Sat, 1 Oct 2011 14:24:49 -0700 Subject: [PATCH 09/13] #361: Removing additional secret key, per CW's request. --- mediagoblin/config_spec.ini | 1 - mediagoblin/middleware/csrf.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 37fe7130..298a6951 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -42,7 +42,6 @@ celery_setup_elsewhere = boolean(default=False) allow_attachments = boolean(default=False) # Cookie stuff -secret_key = string(default="Something Super Duper Secrit!") csrf_cookie_name = string(default='mediagoblin_nonce') [storage:publicstore] diff --git a/mediagoblin/middleware/csrf.py b/mediagoblin/middleware/csrf.py index d41bcd87..44b799d5 100644 --- a/mediagoblin/middleware/csrf.py +++ b/mediagoblin/middleware/csrf.py @@ -106,7 +106,7 @@ class CsrfMiddleware(object): return hashlib.md5("%s%s" % (randrange(0, self.MAX_CSRF_KEY), - mg_globals.app_config['secret_key'])).hexdigest() + randrange(0, self.MAX_CSRF_KEY))).hexdigest() def verify_tokens(self, request): """Verify that the CSRF Cookie exists and that it matches the From 03ae172a60a87625e5281eb9766aa5bf3e37d0f4 Mon Sep 17 00:00:00 2001 From: Aaron Williamson Date: Sat, 1 Oct 2011 18:05:17 -0400 Subject: [PATCH 10/13] Finished splitting util.py into separate files. --- mediagoblin/tools/common.py | 19 +++++ mediagoblin/tools/files.py | 32 +++++++++ mediagoblin/tools/mail.py | 120 ++++++++++++++++++++++++++++++++ mediagoblin/tools/pagination.py | 109 +++++++++++++++++++++++++++++ mediagoblin/tools/request.py | 37 ++++++++++ mediagoblin/tools/response.py | 44 ++++++++++++ mediagoblin/tools/testing.py | 45 ++++++++++++ mediagoblin/tools/text.py | 117 +++++++++++++++++++++++++++++++ 8 files changed, 523 insertions(+) create mode 100644 mediagoblin/tools/files.py create mode 100644 mediagoblin/tools/mail.py create mode 100644 mediagoblin/tools/pagination.py create mode 100644 mediagoblin/tools/request.py create mode 100644 mediagoblin/tools/response.py create mode 100644 mediagoblin/tools/testing.py create mode 100644 mediagoblin/tools/text.py diff --git a/mediagoblin/tools/common.py b/mediagoblin/tools/common.py index dccceccb..ea4541a8 100644 --- a/mediagoblin/tools/common.py +++ b/mediagoblin/tools/common.py @@ -14,5 +14,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import sys + +DISPLAY_IMAGE_FETCHING_ORDER = [u'medium', u'original', u'thumb'] + global TESTS_ENABLED TESTS_ENABLED = False + +def import_component(import_string): + """ + Import a module component defined by STRING. Probably a method, + class, or global variable. + + Args: + - import_string: a string that defines what to import. Written + in the format of "module1.module2:component" + """ + module_name, func_name = import_string.split(':', 1) + __import__(module_name) + module = sys.modules[module_name] + func = getattr(module, func_name) + return func diff --git a/mediagoblin/tools/files.py b/mediagoblin/tools/files.py new file mode 100644 index 00000000..e0bf0569 --- /dev/null +++ b/mediagoblin/tools/files.py @@ -0,0 +1,32 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin import mg_globals + +def delete_media_files(media): + """ + Delete all files associated with a MediaEntry + + Arguments: + - media: A MediaEntry document + """ + for listpath in media['media_files'].itervalues(): + mg_globals.public_store.delete_file( + listpath) + + for attachment in media['attachment_files']: + mg_globals.public_store.delete_file( + attachment['filepath']) diff --git a/mediagoblin/tools/mail.py b/mediagoblin/tools/mail.py new file mode 100644 index 00000000..826acdbf --- /dev/null +++ b/mediagoblin/tools/mail.py @@ -0,0 +1,120 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import smtplib +from email.MIMEText import MIMEText +from mediagoblin import mg_globals +from mediagoblin.tools import common + +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Special email test stuff begins HERE +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# We have two "test inboxes" here: +# +# EMAIL_TEST_INBOX: +# ---------------- +# If you're writing test views, you'll probably want to check this. +# It contains a list of MIMEText messages. +# +# EMAIL_TEST_MBOX_INBOX: +# ---------------------- +# This collects the messages from the FakeMhost inbox. It's reslly +# just here for testing the send_email method itself. +# +# Anyway this contains: +# - from +# - to: a list of email recipient addresses +# - message: not just the body, but the whole message, including +# headers, etc. +# +# ***IMPORTANT!*** +# ---------------- +# Before running tests that call functions which send email, you should +# always call _clear_test_inboxes() to "wipe" the inboxes clean. + +EMAIL_TEST_INBOX = [] +EMAIL_TEST_MBOX_INBOX = [] + +class FakeMhost(object): + """ + Just a fake mail host so we can capture and test messages + from send_email + """ + def login(self, *args, **kwargs): + pass + + def sendmail(self, from_addr, to_addrs, message): + EMAIL_TEST_MBOX_INBOX.append( + {'from': from_addr, + 'to': to_addrs, + 'message': message}) + +def _clear_test_inboxes(): + global EMAIL_TEST_INBOX + global EMAIL_TEST_MBOX_INBOX + EMAIL_TEST_INBOX = [] + EMAIL_TEST_MBOX_INBOX = [] + +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +def send_email(from_addr, to_addrs, subject, message_body): + """ + Simple email sending wrapper, use this so we can capture messages + for unit testing purposes. + + Args: + - from_addr: address you're sending the email from + - to_addrs: list of recipient email addresses + - subject: subject of the email + - message_body: email body text + """ + if common.TESTS_ENABLED or mg_globals.app_config['email_debug_mode']: + mhost = FakeMhost() + elif not mg_globals.app_config['email_debug_mode']: + mhost = smtplib.SMTP( + mg_globals.app_config['email_smtp_host'], + mg_globals.app_config['email_smtp_port']) + + # SMTP.__init__ Issues SMTP.connect implicitly if host + if not mg_globals.app_config['email_smtp_host']: # e.g. host = '' + mhost.connect() # We SMTP.connect explicitly + + if mg_globals.app_config['email_smtp_user'] \ + or mg_globals.app_config['email_smtp_pass']: + mhost.login( + mg_globals.app_config['email_smtp_user'], + mg_globals.app_config['email_smtp_pass']) + + message = MIMEText(message_body.encode('utf-8'), 'plain', 'utf-8') + message['Subject'] = subject + message['From'] = from_addr + message['To'] = ', '.join(to_addrs) + + if common.TESTS_ENABLED: + EMAIL_TEST_INBOX.append(message) + + if mg_globals.app_config['email_debug_mode']: + print u"===== Email =====" + print u"From address: %s" % message['From'] + print u"To addresses: %s" % message['To'] + print u"Subject: %s" % message['Subject'] + print u"-- Body: --" + print message.get_payload(decode=True) + + return mhost.sendmail(from_addr, to_addrs, message.as_string()) diff --git a/mediagoblin/tools/pagination.py b/mediagoblin/tools/pagination.py new file mode 100644 index 00000000..859b60fb --- /dev/null +++ b/mediagoblin/tools/pagination.py @@ -0,0 +1,109 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import urllib +import copy +from math import ceil, floor +from itertools import izip, count + +PAGINATION_DEFAULT_PER_PAGE = 30 + +class Pagination(object): + """ + Pagination class for mongodb queries. + + Initialization through __init__(self, cursor, page=1, per_page=2), + get actual data slice through __call__(). + """ + + def __init__(self, page, cursor, per_page=PAGINATION_DEFAULT_PER_PAGE, + jump_to_id=False): + """ + Initializes Pagination + + Args: + - page: requested page + - per_page: number of objects per page + - cursor: db cursor + - jump_to_id: ObjectId, sets the page to the page containing the object + with _id == jump_to_id. + """ + self.page = page + self.per_page = per_page + self.cursor = cursor + self.total_count = self.cursor.count() + self.active_id = None + + if jump_to_id: + cursor = copy.copy(self.cursor) + + for (doc, increment) in izip(cursor, count(0)): + if doc['_id'] == jump_to_id: + self.page = 1 + int(floor(increment / self.per_page)) + + self.active_id = jump_to_id + break + + + def __call__(self): + """ + Returns slice of objects for the requested page + """ + return self.cursor.skip( + (self.page - 1) * self.per_page).limit(self.per_page) + + @property + def pages(self): + return int(ceil(self.total_count / float(self.per_page))) + + @property + def has_prev(self): + return self.page > 1 + + @property + def has_next(self): + return self.page < self.pages + + def iter_pages(self, left_edge=2, left_current=2, + right_current=5, right_edge=2): + last = 0 + for num in xrange(1, self.pages + 1): + if num <= left_edge or \ + (num > self.page - left_current - 1 and \ + num < self.page + right_current) or \ + num > self.pages - right_edge: + if last + 1 != num: + yield None + yield num + last = num + + def get_page_url_explicit(self, base_url, get_params, page_no): + """ + Get a page url by adding a page= parameter to the base url + """ + new_get_params = copy.copy(get_params or {}) + new_get_params['page'] = page_no + return "%s?%s" % ( + base_url, urllib.urlencode(new_get_params)) + + def get_page_url(self, request, page_no): + """ + Get a new page url based of the request, and the new page number. + + This is a nice wrapper around get_page_url_explicit() + """ + return self.get_page_url_explicit( + request.path_info, request.GET, page_no) diff --git a/mediagoblin/tools/request.py b/mediagoblin/tools/request.py new file mode 100644 index 00000000..b1cbe119 --- /dev/null +++ b/mediagoblin/tools/request.py @@ -0,0 +1,37 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.db.util import ObjectId + +def setup_user_in_request(request): + """ + Examine a request and tack on a request.user parameter if that's + appropriate. + """ + if not request.session.has_key('user_id'): + request.user = None + return + + user = None + user = request.app.db.User.one( + {'_id': ObjectId(request.session['user_id'])}) + + if not user: + # Something's wrong... this user doesn't exist? Invalidate + # this session. + request.session.invalidate() + + request.user = user diff --git a/mediagoblin/tools/response.py b/mediagoblin/tools/response.py new file mode 100644 index 00000000..1477b9bc --- /dev/null +++ b/mediagoblin/tools/response.py @@ -0,0 +1,44 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from webob import Response, exc +from mediagoblin.tools.template import render_template + +def render_to_response(request, template, context, status=200): + """Much like Django's shortcut.render()""" + return Response( + render_template(request, template, context), + status=status) + +def render_404(request): + """ + Render a 404. + """ + return render_to_response( + request, 'mediagoblin/404.html', {}, status=400) + +def redirect(request, *args, **kwargs): + """Returns a HTTPFound(), takes a request and then urlgen params""" + + querystring = None + if kwargs.get('querystring'): + querystring = kwargs.get('querystring') + del kwargs['querystring'] + + return exc.HTTPFound( + location=''.join([ + request.urlgen(*args, **kwargs), + querystring if querystring else ''])) diff --git a/mediagoblin/tools/testing.py b/mediagoblin/tools/testing.py new file mode 100644 index 00000000..39435ca5 --- /dev/null +++ b/mediagoblin/tools/testing.py @@ -0,0 +1,45 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.tools import common +from mediagoblin.tools.template import clear_test_template_context +from mediagoblin.tools.mail import EMAIL_TEST_INBOX, EMAIL_TEST_MBOX_INBOX + +def _activate_testing(): + """ + Call this to activate testing in util.py + """ + + common.TESTS_ENABLED = True + +def clear_test_buckets(): + """ + We store some things for testing purposes that should be cleared + when we want a "clean slate" of information for our next round of + tests. Call this function to wipe all that stuff clean. + + Also wipes out some other things we might redefine during testing, + like the jinja envs. + """ + global SETUP_JINJA_ENVS + SETUP_JINJA_ENVS = {} + + global EMAIL_TEST_INBOX + global EMAIL_TEST_MBOX_INBOX + EMAIL_TEST_INBOX = [] + EMAIL_TEST_MBOX_INBOX = [] + + clear_test_template_context() diff --git a/mediagoblin/tools/text.py b/mediagoblin/tools/text.py new file mode 100644 index 00000000..de4bb281 --- /dev/null +++ b/mediagoblin/tools/text.py @@ -0,0 +1,117 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import wtforms +import markdown +from lxml.html.clean import Cleaner + +from mediagoblin import mg_globals +from mediagoblin.tools import url + +# A super strict version of the lxml.html cleaner class +HTML_CLEANER = Cleaner( + scripts=True, + javascript=True, + comments=True, + style=True, + links=True, + page_structure=True, + processing_instructions=True, + embedded=True, + frames=True, + forms=True, + annoying_tags=True, + allow_tags=[ + 'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br'], + remove_unknown_tags=False, # can't be used with allow_tags + safe_attrs_only=True, + add_nofollow=True, # for now + host_whitelist=(), + whitelist_tags=set([])) + +def clean_html(html): + # clean_html barfs on an empty string + if not html: + return u'' + + return HTML_CLEANER.clean_html(html) + +def convert_to_tag_list_of_dicts(tag_string): + """ + Filter input from incoming string containing user tags, + + Strips trailing, leading, and internal whitespace, and also converts + the "tags" text into an array of tags + """ + taglist = [] + if tag_string: + + # Strip out internal, trailing, and leading whitespace + stripped_tag_string = u' '.join(tag_string.strip().split()) + + # Split the tag string into a list of tags + for tag in stripped_tag_string.split( + mg_globals.app_config['tags_delimiter']): + + # Ignore empty or duplicate tags + if tag.strip() and tag.strip() not in [t['name'] for t in taglist]: + + taglist.append({'name': tag.strip(), + 'slug': url.slugify(tag.strip())}) + return taglist + +def media_tags_as_string(media_entry_tags): + """ + Generate a string from a media item's tags, stored as a list of dicts + + This is the opposite of convert_to_tag_list_of_dicts + """ + media_tag_string = '' + if media_entry_tags: + media_tag_string = mg_globals.app_config['tags_delimiter'].join( + [tag['name'] for tag in media_entry_tags]) + return media_tag_string + +TOO_LONG_TAG_WARNING = \ + u'Tags must be shorter than %s characters. Tags that are too long: %s' + +def tag_length_validator(form, field): + """ + Make sure tags do not exceed the maximum tag length. + """ + tags = convert_to_tag_list_of_dicts(field.data) + too_long_tags = [ + tag['name'] for tag in tags + if len(tag['name']) > mg_globals.app_config['tags_max_length']] + + if too_long_tags: + raise wtforms.ValidationError( + TOO_LONG_TAG_WARNING % (mg_globals.app_config['tags_max_length'], \ + ', '.join(too_long_tags))) + + +MARKDOWN_INSTANCE = markdown.Markdown(safe_mode='escape') + +def cleaned_markdown_conversion(text): + """ + Take a block of text, run it through MarkDown, and clean its HTML. + """ + # Markdown will do nothing with and clean_html can do nothing with + # an empty string :) + if not text: + return u'' + + return clean_html(MARKDOWN_INSTANCE.convert(text)) From 152a3bfaa36d58e44979f217c5799531f780250f Mon Sep 17 00:00:00 2001 From: Aaron Williamson Date: Sat, 1 Oct 2011 18:05:44 -0400 Subject: [PATCH 11/13] Finished splitting util.py into separate files. --- mediagoblin/app.py | 13 +- mediagoblin/auth/lib.py | 2 +- mediagoblin/auth/views.py | 2 +- mediagoblin/db/migrations.py | 2 +- mediagoblin/db/models.py | 12 +- mediagoblin/decorators.py | 2 +- mediagoblin/edit/forms.py | 6 +- mediagoblin/edit/views.py | 11 +- mediagoblin/listings/views.py | 3 +- mediagoblin/storage/__init__.py | 4 +- mediagoblin/submit/forms.py | 2 +- mediagoblin/submit/views.py | 5 +- mediagoblin/tests/test_auth.py | 10 +- mediagoblin/tests/test_tags.py | 11 +- mediagoblin/tests/test_util.py | 23 +- mediagoblin/tests/tools.py | 6 +- mediagoblin/user_pages/views.py | 7 +- mediagoblin/util.py | 699 -------------------------------- mediagoblin/views.py | 3 +- 19 files changed, 61 insertions(+), 762 deletions(-) delete mode 100644 mediagoblin/util.py diff --git a/mediagoblin/app.py b/mediagoblin/app.py index 5ee3b973..0f25a4e5 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -20,8 +20,9 @@ import urllib import routes from webob import Request, exc -from mediagoblin import routing, util, middleware -from mediagoblin.tools import translate, template +from mediagoblin import routing, middleware +from mediagoblin.tools import common, translate, template, response +from mediagoblin.tools import request as mg_request from mediagoblin.mg_globals import setup_globals from mediagoblin.init.celery import setup_celery_from_config from mediagoblin.init import (get_jinja_loader, get_staticdirector, @@ -99,7 +100,7 @@ class MediaGoblinApp(object): setup_workbench() # instantiate application middleware - self.middleware = [util.import_component(m)(self) + self.middleware = [common.import_component(m)(self) for m in middleware.ENABLED_MIDDLEWARE] @@ -131,7 +132,7 @@ class MediaGoblinApp(object): request.db = self.db request.staticdirect = self.staticdirector - util.setup_user_in_request(request) + mg_request.setup_user_in_request(request) # No matching page? if route_match is None: @@ -149,9 +150,9 @@ class MediaGoblinApp(object): # Okay, no matches. 404 time! request.matchdict = {} # in case our template expects it - return util.render_404(request)(environ, start_response) + return response.render_404(request)(environ, start_response) - controller = util.import_component(route_match['controller']) + controller = common.import_component(route_match['controller']) request.start_response = start_response # get the response from the controller diff --git a/mediagoblin/auth/lib.py b/mediagoblin/auth/lib.py index bf5a2399..4c57ef88 100644 --- a/mediagoblin/auth/lib.py +++ b/mediagoblin/auth/lib.py @@ -19,7 +19,7 @@ import random import bcrypt -from mediagoblin.util import send_email +from mediagoblin.tools.mail import send_email from mediagoblin.tools.template import render_template from mediagoblin import mg_globals diff --git a/mediagoblin/auth/views.py b/mediagoblin/auth/views.py index 9bfa93cf..88c91565 100644 --- a/mediagoblin/auth/views.py +++ b/mediagoblin/auth/views.py @@ -21,7 +21,7 @@ from webob import exc from mediagoblin import messages from mediagoblin import mg_globals -from mediagoblin.util import render_to_response, redirect, render_404 +from mediagoblin.tools.response import render_to_response, redirect, render_404 from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.db.util import ObjectId, InvalidId from mediagoblin.auth import lib as auth_lib diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 755f49c5..3cafe4f8 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from mediagoblin.db.util import RegisterMigration -from mediagoblin.util import cleaned_markdown_conversion +from mediagoblin.tools.text import cleaned_markdown_conversion # Please see mediagoblin/tests/test_migrations.py for some examples of diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index eacc801c..0f5174cc 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -18,14 +18,12 @@ import datetime, uuid from mongokit import Document -from mediagoblin import util from mediagoblin.auth import lib as auth_lib from mediagoblin import mg_globals from mediagoblin.db import migrations from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId -from mediagoblin.util import Pagination -from mediagoblin.util import DISPLAY_IMAGE_FETCHING_ORDER -from mediagoblin.tools import url +from mediagoblin.tools.pagination import Pagination +from mediagoblin.tools import url, common ################### # Custom validators @@ -220,7 +218,7 @@ class MediaEntry(Document): return self.db.MediaComment.find({ 'media_entry': self['_id']}).sort('created', DESCENDING) - def get_display_media(self, media_map, fetch_order=DISPLAY_IMAGE_FETCHING_ORDER): + def get_display_media(self, media_map, fetch_order=common.DISPLAY_IMAGE_FETCHING_ORDER): """ Find the best media for display. @@ -234,7 +232,7 @@ class MediaEntry(Document): """ media_sizes = media_map.keys() - for media_size in DISPLAY_IMAGE_FETCHING_ORDER: + for media_size in common.DISPLAY_IMAGE_FETCHING_ORDER: if media_size in media_sizes: return media_map[media_size] @@ -304,7 +302,7 @@ class MediaEntry(Document): Get the exception that's appropriate for this error """ if self['fail_error']: - return util.import_component(self['fail_error']) + return common.import_component(self['fail_error']) class MediaComment(Document): diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index 7d5978fc..19e22bca 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -17,7 +17,7 @@ from webob import exc -from mediagoblin.util import redirect, render_404 +from mediagoblin.tools.response import redirect, render_404 from mediagoblin.db.util import ObjectId, InvalidId diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index f81d58b2..7e71722c 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -14,12 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - import wtforms -from mediagoblin.util import tag_length_validator, TOO_LONG_TAG_WARNING -from mediagoblin.util import fake_ugettext_passthrough as _ - +from mediagoblin.tools.text import tag_length_validator, TOO_LONG_TAG_WARNING +from mediagoblin.tools.translate import fake_ugettext_passthrough as _ class EditForm(wtforms.Form): title = wtforms.TextField( diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 15edfdd6..a6ddb553 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -25,14 +25,15 @@ from werkzeug.utils import secure_filename from mediagoblin import messages from mediagoblin import mg_globals -from mediagoblin.util import ( - render_to_response, redirect, clean_html, convert_to_tag_list_of_dicts, - media_tags_as_string, cleaned_markdown_conversion) -from mediagoblin.util import pass_to_ugettext as _ + from mediagoblin.edit import forms from mediagoblin.edit.lib import may_edit_media from mediagoblin.decorators import require_active_login, get_user_media_entry - +from mediagoblin.tools.response import render_to_response, redirect +from mediagoblin.tools.translate import pass_to_ugettext as _ +from mediagoblin.tools.text import ( + clean_html, convert_to_tag_list_of_dicts, + media_tags_as_string, cleaned_markdown_conversion) @get_user_media_entry @require_active_login diff --git a/mediagoblin/listings/views.py b/mediagoblin/listings/views.py index b3384eb4..01aad803 100644 --- a/mediagoblin/listings/views.py +++ b/mediagoblin/listings/views.py @@ -16,7 +16,8 @@ from mediagoblin.db.util import DESCENDING -from mediagoblin.util import Pagination, render_to_response +from mediagoblin.tools.pagination import Pagination +from mediagoblin.tools.response import render_to_response from mediagoblin.decorators import uses_pagination from werkzeug.contrib.atom import AtomFeed diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 8665d9e5..9e592b9e 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -21,7 +21,7 @@ import uuid from werkzeug.utils import secure_filename -from mediagoblin import util +from mediagoblin.tools import common ######## # Errors @@ -236,5 +236,5 @@ def storage_system_from_config(config_section): else: storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage' - storage_class = util.import_component(storage_class) + storage_class = common.import_component(storage_class) return storage_class(**config_params) diff --git a/mediagoblin/submit/forms.py b/mediagoblin/submit/forms.py index 200ce4e4..25d6e304 100644 --- a/mediagoblin/submit/forms.py +++ b/mediagoblin/submit/forms.py @@ -17,7 +17,7 @@ import wtforms -from mediagoblin.util import tag_length_validator +from mediagoblin.tools.text import tag_length_validator from mediagoblin.tools.translate import fake_ugettext_passthrough as _ diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index cd34e006..7134235e 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -22,10 +22,9 @@ from cgi import FieldStorage from werkzeug.utils import secure_filename from mediagoblin.db.util import ObjectId -from mediagoblin.util import ( - render_to_response, redirect, cleaned_markdown_conversion, \ - convert_to_tag_list_of_dicts) +from mediagoblin.tools.text import cleaned_markdown_conversion, convert_to_tag_list_of_dicts from mediagoblin.tools.translate import pass_to_ugettext as _ +from mediagoblin.tools.response import render_to_response, redirect from mediagoblin.decorators import require_active_login from mediagoblin.submit import forms as submit_forms, security from mediagoblin.process_media import process_media, mark_entry_failed diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index f00456c4..40961eca 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -22,7 +22,7 @@ from nose.tools import assert_equal from mediagoblin.auth import lib as auth_lib from mediagoblin.tests.tools import setup_fresh_app from mediagoblin import mg_globals -from mediagoblin.tools import template +from mediagoblin.tools import template, mail ######################## @@ -171,8 +171,8 @@ def test_register_views(test_app): assert request.session['user_id'] == unicode(new_user['_id']) ## Make sure we get email confirmation, and try verifying - assert len(template.EMAIL_TEST_INBOX) == 1 - message = template.EMAIL_TEST_INBOX.pop() + assert len(mail.EMAIL_TEST_INBOX) == 1 + message = mail.EMAIL_TEST_INBOX.pop() assert message['To'] == 'happygrrl@example.org' email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/verification_email.txt'] @@ -254,8 +254,8 @@ def test_register_views(test_app): 'mediagoblin/auth/fp_email_sent.html') ## Make sure link to change password is sent by email - assert len(template.EMAIL_TEST_INBOX) == 1 - message = template.EMAIL_TEST_INBOX.pop() + assert len(mail.EMAIL_TEST_INBOX) == 1 + message = mail.EMAIL_TEST_INBOX.pop() assert message['To'] == 'happygrrl@example.org' email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/fp_verification_email.txt'] diff --git a/mediagoblin/tests/test_tags.py b/mediagoblin/tests/test_tags.py index d4628795..a05831c9 100644 --- a/mediagoblin/tests/test_tags.py +++ b/mediagoblin/tests/test_tags.py @@ -15,9 +15,8 @@ # along with this program. If not, see . from mediagoblin.tests.tools import setup_fresh_app -from mediagoblin import util from mediagoblin import mg_globals - +from mediagoblin.tools import text @setup_fresh_app def test_list_of_dicts_conversion(test_app): @@ -28,23 +27,23 @@ def test_list_of_dicts_conversion(test_app): function performs the reverse operation when populating a form to edit tags. """ # Leading, trailing, and internal whitespace should be removed and slugified - assert util.convert_to_tag_list_of_dicts('sleep , 6 AM, chainsaw! ') == [ + assert text.convert_to_tag_list_of_dicts('sleep , 6 AM, chainsaw! ') == [ {'name': u'sleep', 'slug': u'sleep'}, {'name': u'6 AM', 'slug': u'6-am'}, {'name': u'chainsaw!', 'slug': u'chainsaw'}] # If the user enters two identical tags, record only one of them - assert util.convert_to_tag_list_of_dicts('echo,echo') == [{'name': u'echo', + assert text.convert_to_tag_list_of_dicts('echo,echo') == [{'name': u'echo', 'slug': u'echo'}] # Make sure converting the list of dicts to a string works - assert util.media_tags_as_string([{'name': u'yin', 'slug': u'yin'}, + assert text.media_tags_as_string([{'name': u'yin', 'slug': u'yin'}, {'name': u'yang', 'slug': u'yang'}]) == \ u'yin,yang' # If the tag delimiter is a space then we expect different results mg_globals.app_config['tags_delimiter'] = u' ' - assert util.convert_to_tag_list_of_dicts('unicorn ceramic nazi') == [ + assert text.convert_to_tag_list_of_dicts('unicorn ceramic nazi') == [ {'name': u'unicorn', 'slug': u'unicorn'}, {'name': u'ceramic', 'slug': u'ceramic'}, {'name': u'nazi', 'slug': u'nazi'}] diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index cdc62b7d..48fa8669 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -16,10 +16,9 @@ import email -from mediagoblin import util -from mediagoblin.tools import url, translate +from mediagoblin.tools import common, url, translate, mail, text, testing -util._activate_testing() +testing._activate_testing() def _import_component_testing_method(silly_string): @@ -28,7 +27,7 @@ def _import_component_testing_method(silly_string): def test_import_component(): - imported_func = util.import_component( + imported_func = common.import_component( 'mediagoblin.tests.test_util:_import_component_testing_method') result = imported_func('hooobaladoobala') expected = u"'hooobaladoobala' is the silliest string I've ever seen" @@ -36,10 +35,10 @@ def test_import_component(): def test_send_email(): - util._clear_test_inboxes() + mail._clear_test_inboxes() # send the email - util.send_email( + mail.send_email( "sender@mediagoblin.example.org", ["amanda@example.org", "akila@example.org"], "Testing is so much fun!", @@ -48,8 +47,8 @@ def test_send_email(): I hope you like unit tests JUST AS MUCH AS I DO!""") # check the main inbox - assert len(util.EMAIL_TEST_INBOX) == 1 - message = util.EMAIL_TEST_INBOX.pop() + assert len(mail.EMAIL_TEST_INBOX) == 1 + message = mail.EMAIL_TEST_INBOX.pop() assert message['From'] == "sender@mediagoblin.example.org" assert message['To'] == "amanda@example.org, akila@example.org" assert message['Subject'] == "Testing is so much fun!" @@ -58,8 +57,8 @@ I hope you like unit tests JUST AS MUCH AS I DO!""") I hope you like unit tests JUST AS MUCH AS I DO!""" # Check everything that the FakeMhost.sendmail() method got is correct - assert len(util.EMAIL_TEST_MBOX_INBOX) == 1 - mbox_dict = util.EMAIL_TEST_MBOX_INBOX.pop() + assert len(mail.EMAIL_TEST_MBOX_INBOX) == 1 + mbox_dict = mail.EMAIL_TEST_MBOX_INBOX.pop() assert mbox_dict['from'] == "sender@mediagoblin.example.org" assert mbox_dict['to'] == ["amanda@example.org", "akila@example.org"] mbox_message = email.message_from_string(mbox_dict['message']) @@ -107,7 +106,7 @@ def test_locale_to_lower_lower(): def test_html_cleaner(): # Remove images - result = util.clean_html( + result = text.clean_html( '

Hi everybody! ' '

\n' '

:)

') @@ -118,7 +117,7 @@ def test_html_cleaner(): '') # Remove evil javascript - result = util.clean_html( + result = text.clean_html( '

innocent link!

') assert result == ( '

innocent link!

') diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 308e83ee..cf84da14 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -21,7 +21,7 @@ import os, shutil from paste.deploy import loadapp from webtest import TestApp -from mediagoblin import util +from mediagoblin.tools import testing from mediagoblin.init.config import read_mediagoblin_config from mediagoblin.decorators import _make_safe from mediagoblin.db.open import setup_connection_and_db_from_config @@ -59,7 +59,7 @@ def get_test_app(dump_old_app=True): suicide_if_bad_celery_environ() # Make sure we've turned on testing - util._activate_testing() + testing._activate_testing() # Leave this imported as it sets up celery. from mediagoblin.init.celery import from_tests @@ -117,7 +117,7 @@ def setup_fresh_app(func): """ def wrapper(*args, **kwargs): test_app = get_test_app() - util.clear_test_buckets() + testing.clear_test_buckets() return func(test_app, *args, **kwargs) return _make_safe(wrapper, func) diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 40c7ffce..9cec74dc 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -18,10 +18,11 @@ from webob import exc from mediagoblin import messages, mg_globals from mediagoblin.db.util import DESCENDING, ObjectId -from mediagoblin.util import ( - Pagination, render_to_response, redirect, cleaned_markdown_conversion, - render_404, delete_media_files) +from mediagoblin.tools.text import cleaned_markdown_conversion +from mediagoblin.tools.response import render_to_response, render_404, redirect from mediagoblin.tools.translate import pass_to_ugettext as _ +from mediagoblin.tools.pagination import Pagination +from mediagoblin.tools.files import delete_media_files from mediagoblin.user_pages import forms as user_forms from mediagoblin.decorators import (uses_pagination, get_user_media_entry, diff --git a/mediagoblin/util.py b/mediagoblin/util.py deleted file mode 100644 index 35755ccf..00000000 --- a/mediagoblin/util.py +++ /dev/null @@ -1,699 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from __future__ import division - -from email.MIMEText import MIMEText -#import gettext -#import pkg_resources -import smtplib -import sys -#import re -#import translitcodec -import urllib -from math import ceil, floor -import copy -import wtforms - -#from babel.localedata import exists -#from babel.support import LazyProxy -#import jinja2 -from webob import Response, exc -from lxml.html.clean import Cleaner -import markdown -from wtforms.form import Form - -from mediagoblin import mg_globals -#from mediagoblin import messages -from mediagoblin.db.util import ObjectId -from mediagoblin.tools import url -from mediagoblin.tools import common -from mediagoblin.tools.template import TEMPLATE_TEST_CONTEXT, render_template - -from itertools import izip, count - -DISPLAY_IMAGE_FETCHING_ORDER = [u'medium', u'original', u'thumb'] - -def _activate_testing(): - """ - Call this to activate testing in util.py - """ - - common.TESTS_ENABLED = True - -def clear_test_buckets(): - """ - We store some things for testing purposes that should be cleared - when we want a "clean slate" of information for our next round of - tests. Call this function to wipe all that stuff clean. - - Also wipes out some other things we might redefine during testing, - like the jinja envs. - """ - global SETUP_JINJA_ENVS - SETUP_JINJA_ENVS = {} - - global EMAIL_TEST_INBOX - global EMAIL_TEST_MBOX_INBOX - EMAIL_TEST_INBOX = [] - EMAIL_TEST_MBOX_INBOX = [] - - clear_test_template_context() - - -# SETUP_JINJA_ENVS = {} - - -# def get_jinja_env(template_loader, locale): -# """ -# Set up the Jinja environment, - -# (In the future we may have another system for providing theming; -# for now this is good enough.) -# """ -# setup_gettext(locale) - -# # If we have a jinja environment set up with this locale, just -# # return that one. -# if SETUP_JINJA_ENVS.has_key(locale): -# return SETUP_JINJA_ENVS[locale] - -# template_env = jinja2.Environment( -# loader=template_loader, autoescape=True, -# extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape']) - -# template_env.install_gettext_callables( -# mg_globals.translations.ugettext, -# mg_globals.translations.ungettext) - -# # All templates will know how to ... -# # ... fetch all waiting messages and remove them from the queue -# # ... construct a grid of thumbnails or other media -# template_env.globals['fetch_messages'] = messages.fetch_messages -# template_env.globals['gridify_list'] = gridify_list -# template_env.globals['gridify_cursor'] = gridify_cursor - -# if exists(locale): -# SETUP_JINJA_ENVS[locale] = template_env - -# return template_env - - -# # We'll store context information here when doing unit tests -# TEMPLATE_TEST_CONTEXT = {} - - -# def render_template(request, template_path, context): -# """ -# Render a template with context. - -# Always inserts the request into the context, so you don't have to. -# Also stores the context if we're doing unit tests. Helpful! -# """ -# template = request.template_env.get_template( -# template_path) -# context['request'] = request -# rendered = template.render(context) - -# if TESTS_ENABLED: -# TEMPLATE_TEST_CONTEXT[template_path] = context - -# return rendered - - -def clear_test_template_context(): - global TEMPLATE_TEST_CONTEXT - TEMPLATE_TEST_CONTEXT = {} - - -def render_to_response(request, template, context, status=200): - """Much like Django's shortcut.render()""" - return Response( - render_template(request, template, context), - status=status) - - -def redirect(request, *args, **kwargs): - """Returns a HTTPFound(), takes a request and then urlgen params""" - - querystring = None - if kwargs.get('querystring'): - querystring = kwargs.get('querystring') - del kwargs['querystring'] - - return exc.HTTPFound( - location=''.join([ - request.urlgen(*args, **kwargs), - querystring if querystring else ''])) - - -def setup_user_in_request(request): - """ - Examine a request and tack on a request.user parameter if that's - appropriate. - """ - if not request.session.has_key('user_id'): - request.user = None - return - - user = None - user = request.app.db.User.one( - {'_id': ObjectId(request.session['user_id'])}) - - if not user: - # Something's wrong... this user doesn't exist? Invalidate - # this session. - request.session.invalidate() - - request.user = user - - -def import_component(import_string): - """ - Import a module component defined by STRING. Probably a method, - class, or global variable. - - Args: - - import_string: a string that defines what to import. Written - in the format of "module1.module2:component" - """ - module_name, func_name = import_string.split(':', 1) - __import__(module_name) - module = sys.modules[module_name] - func = getattr(module, func_name) - return func - -# _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') - -# def slugify(text, delim=u'-'): -# """ -# Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/ -# """ -# result = [] -# for word in _punct_re.split(text.lower()): -# word = word.encode('translit/long') -# if word: -# result.append(word) -# return unicode(delim.join(result)) - -### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -### Special email test stuff begins HERE -### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# We have two "test inboxes" here: -# -# EMAIL_TEST_INBOX: -# ---------------- -# If you're writing test views, you'll probably want to check this. -# It contains a list of MIMEText messages. -# -# EMAIL_TEST_MBOX_INBOX: -# ---------------------- -# This collects the messages from the FakeMhost inbox. It's reslly -# just here for testing the send_email method itself. -# -# Anyway this contains: -# - from -# - to: a list of email recipient addresses -# - message: not just the body, but the whole message, including -# headers, etc. -# -# ***IMPORTANT!*** -# ---------------- -# Before running tests that call functions which send email, you should -# always call _clear_test_inboxes() to "wipe" the inboxes clean. - -EMAIL_TEST_INBOX = [] -EMAIL_TEST_MBOX_INBOX = [] - - -class FakeMhost(object): - """ - Just a fake mail host so we can capture and test messages - from send_email - """ - def login(self, *args, **kwargs): - pass - - def sendmail(self, from_addr, to_addrs, message): - EMAIL_TEST_MBOX_INBOX.append( - {'from': from_addr, - 'to': to_addrs, - 'message': message}) - -def _clear_test_inboxes(): - global EMAIL_TEST_INBOX - global EMAIL_TEST_MBOX_INBOX - EMAIL_TEST_INBOX = [] - EMAIL_TEST_MBOX_INBOX = [] - -### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -### -### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -def send_email(from_addr, to_addrs, subject, message_body): - """ - Simple email sending wrapper, use this so we can capture messages - for unit testing purposes. - - Args: - - from_addr: address you're sending the email from - - to_addrs: list of recipient email addresses - - subject: subject of the email - - message_body: email body text - """ - if common.TESTS_ENABLED or mg_globals.app_config['email_debug_mode']: - mhost = FakeMhost() - elif not mg_globals.app_config['email_debug_mode']: - mhost = smtplib.SMTP( - mg_globals.app_config['email_smtp_host'], - mg_globals.app_config['email_smtp_port']) - - # SMTP.__init__ Issues SMTP.connect implicitly if host - if not mg_globals.app_config['email_smtp_host']: # e.g. host = '' - mhost.connect() # We SMTP.connect explicitly - - if mg_globals.app_config['email_smtp_user'] \ - or mg_globals.app_config['email_smtp_pass']: - mhost.login( - mg_globals.app_config['email_smtp_user'], - mg_globals.app_config['email_smtp_pass']) - - message = MIMEText(message_body.encode('utf-8'), 'plain', 'utf-8') - message['Subject'] = subject - message['From'] = from_addr - message['To'] = ', '.join(to_addrs) - - if common.TESTS_ENABLED: - EMAIL_TEST_INBOX.append(message) - - if mg_globals.app_config['email_debug_mode']: - print u"===== Email =====" - print u"From address: %s" % message['From'] - print u"To addresses: %s" % message['To'] - print u"Subject: %s" % message['Subject'] - print u"-- Body: --" - print message.get_payload(decode=True) - - return mhost.sendmail(from_addr, to_addrs, message.as_string()) - - -# ################### -# # Translation tools -# ################### - - -# TRANSLATIONS_PATH = pkg_resources.resource_filename( -# 'mediagoblin', 'i18n') - - -# def locale_to_lower_upper(locale): -# """ -# Take a locale, regardless of style, and format it like "en-us" -# """ -# if '-' in locale: -# lang, country = locale.split('-', 1) -# return '%s_%s' % (lang.lower(), country.upper()) -# elif '_' in locale: -# lang, country = locale.split('_', 1) -# return '%s_%s' % (lang.lower(), country.upper()) -# else: -# return locale.lower() - - -# def locale_to_lower_lower(locale): -# """ -# Take a locale, regardless of style, and format it like "en_US" -# """ -# if '_' in locale: -# lang, country = locale.split('_', 1) -# return '%s-%s' % (lang.lower(), country.lower()) -# else: -# return locale.lower() - - -# def get_locale_from_request(request): -# """ -# Figure out what target language is most appropriate based on the -# request -# """ -# request_form = request.GET or request.POST - -# if request_form.has_key('lang'): -# return locale_to_lower_upper(request_form['lang']) - -# accept_lang_matches = request.accept_language.best_matches() - -# # Your routing can explicitly specify a target language -# matchdict = request.matchdict or {} - -# if matchdict.has_key('locale'): -# target_lang = matchdict['locale'] -# elif request.session.has_key('target_lang'): -# target_lang = request.session['target_lang'] -# # Pull the first acceptable language -# elif accept_lang_matches: -# target_lang = accept_lang_matches[0] -# # Fall back to English -# else: -# target_lang = 'en' - -# return locale_to_lower_upper(target_lang) - - -# A super strict version of the lxml.html cleaner class -HTML_CLEANER = Cleaner( - scripts=True, - javascript=True, - comments=True, - style=True, - links=True, - page_structure=True, - processing_instructions=True, - embedded=True, - frames=True, - forms=True, - annoying_tags=True, - allow_tags=[ - 'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br'], - remove_unknown_tags=False, # can't be used with allow_tags - safe_attrs_only=True, - add_nofollow=True, # for now - host_whitelist=(), - whitelist_tags=set([])) - - -def clean_html(html): - # clean_html barfs on an empty string - if not html: - return u'' - - return HTML_CLEANER.clean_html(html) - - -def convert_to_tag_list_of_dicts(tag_string): - """ - Filter input from incoming string containing user tags, - - Strips trailing, leading, and internal whitespace, and also converts - the "tags" text into an array of tags - """ - taglist = [] - if tag_string: - - # Strip out internal, trailing, and leading whitespace - stripped_tag_string = u' '.join(tag_string.strip().split()) - - # Split the tag string into a list of tags - for tag in stripped_tag_string.split( - mg_globals.app_config['tags_delimiter']): - - # Ignore empty or duplicate tags - if tag.strip() and tag.strip() not in [t['name'] for t in taglist]: - - taglist.append({'name': tag.strip(), - 'slug': url.slugify(tag.strip())}) - return taglist - - -def media_tags_as_string(media_entry_tags): - """ - Generate a string from a media item's tags, stored as a list of dicts - - This is the opposite of convert_to_tag_list_of_dicts - """ - media_tag_string = '' - if media_entry_tags: - media_tag_string = mg_globals.app_config['tags_delimiter'].join( - [tag['name'] for tag in media_entry_tags]) - return media_tag_string - -TOO_LONG_TAG_WARNING = \ - u'Tags must be shorter than %s characters. Tags that are too long: %s' - -def tag_length_validator(form, field): - """ - Make sure tags do not exceed the maximum tag length. - """ - tags = convert_to_tag_list_of_dicts(field.data) - too_long_tags = [ - tag['name'] for tag in tags - if len(tag['name']) > mg_globals.app_config['tags_max_length']] - - if too_long_tags: - raise wtforms.ValidationError( - TOO_LONG_TAG_WARNING % (mg_globals.app_config['tags_max_length'], \ - ', '.join(too_long_tags))) - - -MARKDOWN_INSTANCE = markdown.Markdown(safe_mode='escape') - -def cleaned_markdown_conversion(text): - """ - Take a block of text, run it through MarkDown, and clean its HTML. - """ - # Markdown will do nothing with and clean_html can do nothing with - # an empty string :) - if not text: - return u'' - - return clean_html(MARKDOWN_INSTANCE.convert(text)) - - -# SETUP_GETTEXTS = {} - -# def setup_gettext(locale): -# """ -# Setup the gettext instance based on this locale -# """ -# # Later on when we have plugins we may want to enable the -# # multi-translations system they have so we can handle plugin -# # translations too - -# # TODO: fallback nicely on translations from pt_PT to pt if not -# # available, etc. -# if SETUP_GETTEXTS.has_key(locale): -# this_gettext = SETUP_GETTEXTS[locale] -# else: -# this_gettext = gettext.translation( -# 'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True) -# if exists(locale): -# SETUP_GETTEXTS[locale] = this_gettext - -# mg_globals.setup_globals( -# translations=this_gettext) - - -# # Force en to be setup before anything else so that -# # mg_globals.translations is never None -# setup_gettext('en') - - -# def pass_to_ugettext(*args, **kwargs): -# """ -# Pass a translation on to the appropriate ugettext method. - -# The reason we can't have a global ugettext method is because -# mg_globals gets swapped out by the application per-request. -# """ -# return mg_globals.translations.ugettext( -# *args, **kwargs) - - -# def lazy_pass_to_ugettext(*args, **kwargs): -# """ -# Lazily pass to ugettext. - -# This is useful if you have to define a translation on a module -# level but you need it to not translate until the time that it's -# used as a string. -# """ -# return LazyProxy(pass_to_ugettext, *args, **kwargs) - - -# def pass_to_ngettext(*args, **kwargs): -# """ -# Pass a translation on to the appropriate ngettext method. - -# The reason we can't have a global ngettext method is because -# mg_globals gets swapped out by the application per-request. -# """ -# return mg_globals.translations.ngettext( -# *args, **kwargs) - - -# def lazy_pass_to_ngettext(*args, **kwargs): -# """ -# Lazily pass to ngettext. - -# This is useful if you have to define a translation on a module -# level but you need it to not translate until the time that it's -# used as a string. -# """ -# return LazyProxy(pass_to_ngettext, *args, **kwargs) - - -# def fake_ugettext_passthrough(string): -# """ -# Fake a ugettext call for extraction's sake ;) - -# In wtforms there's a separate way to define a method to translate -# things... so we just need to mark up the text so that it can be -# extracted, not so that it's actually run through gettext. -# """ -# return string - - -PAGINATION_DEFAULT_PER_PAGE = 30 - -class Pagination(object): - """ - Pagination class for mongodb queries. - - Initialization through __init__(self, cursor, page=1, per_page=2), - get actual data slice through __call__(). - """ - - def __init__(self, page, cursor, per_page=PAGINATION_DEFAULT_PER_PAGE, - jump_to_id=False): - """ - Initializes Pagination - - Args: - - page: requested page - - per_page: number of objects per page - - cursor: db cursor - - jump_to_id: ObjectId, sets the page to the page containing the object - with _id == jump_to_id. - """ - self.page = page - self.per_page = per_page - self.cursor = cursor - self.total_count = self.cursor.count() - self.active_id = None - - if jump_to_id: - cursor = copy.copy(self.cursor) - - for (doc, increment) in izip(cursor, count(0)): - if doc['_id'] == jump_to_id: - self.page = 1 + int(floor(increment / self.per_page)) - - self.active_id = jump_to_id - break - - - def __call__(self): - """ - Returns slice of objects for the requested page - """ - return self.cursor.skip( - (self.page - 1) * self.per_page).limit(self.per_page) - - @property - def pages(self): - return int(ceil(self.total_count / float(self.per_page))) - - @property - def has_prev(self): - return self.page > 1 - - @property - def has_next(self): - return self.page < self.pages - - def iter_pages(self, left_edge=2, left_current=2, - right_current=5, right_edge=2): - last = 0 - for num in xrange(1, self.pages + 1): - if num <= left_edge or \ - (num > self.page - left_current - 1 and \ - num < self.page + right_current) or \ - num > self.pages - right_edge: - if last + 1 != num: - yield None - yield num - last = num - - def get_page_url_explicit(self, base_url, get_params, page_no): - """ - Get a page url by adding a page= parameter to the base url - """ - new_get_params = copy.copy(get_params or {}) - new_get_params['page'] = page_no - return "%s?%s" % ( - base_url, urllib.urlencode(new_get_params)) - - def get_page_url(self, request, page_no): - """ - Get a new page url based of the request, and the new page number. - - This is a nice wrapper around get_page_url_explicit() - """ - return self.get_page_url_explicit( - request.path_info, request.GET, page_no) - - -# def gridify_list(this_list, num_cols=5): -# """ -# Generates a list of lists where each sub-list's length depends on -# the number of columns in the list -# """ -# grid = [] - -# # Figure out how many rows we should have -# num_rows = int(ceil(float(len(this_list)) / num_cols)) - -# for row_num in range(num_rows): -# slice_min = row_num * num_cols -# slice_max = (row_num + 1) * num_cols - -# row = this_list[slice_min:slice_max] - -# grid.append(row) - -# return grid - - -# def gridify_cursor(this_cursor, num_cols=5): -# """ -# Generates a list of lists where each sub-list's length depends on -# the number of columns in the list -# """ -# return gridify_list(list(this_cursor), num_cols) - - -def render_404(request): - """ - Render a 404. - """ - return render_to_response( - request, 'mediagoblin/404.html', {}, status=400) - -def delete_media_files(media): - """ - Delete all files associated with a MediaEntry - - Arguments: - - media: A MediaEntry document - """ - for listpath in media['media_files'].itervalues(): - mg_globals.public_store.delete_file( - listpath) - - for attachment in media['attachment_files']: - mg_globals.public_store.delete_file( - attachment['filepath']) diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 96687f96..22f9268d 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -15,7 +15,8 @@ # along with this program. If not, see . from mediagoblin import mg_globals -from mediagoblin.util import render_to_response, Pagination +from mediagoblin.tools.pagination import Pagination +from mediagoblin.tools.response import render_to_response from mediagoblin.db.util import DESCENDING from mediagoblin.decorators import uses_pagination From 88233cb282ceacd4c9e8cac419faf952627e7fe2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 1 Oct 2011 21:30:41 -0500 Subject: [PATCH 12/13] Fixing indentation in start.html --- mediagoblin/templates/mediagoblin/submit/start.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/templates/mediagoblin/submit/start.html b/mediagoblin/templates/mediagoblin/submit/start.html index 7bc6ff45..29b01181 100644 --- a/mediagoblin/templates/mediagoblin/submit/start.html +++ b/mediagoblin/templates/mediagoblin/submit/start.html @@ -26,8 +26,8 @@

{% trans %}Submit yer media{% endtrans %}

{{ wtforms_util.render_divs(submit_form) }}
- {{ csrf_token }} - + {{ csrf_token }} +
From 4d7a93a49303344830021bab5a741148b1adb7c3 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 1 Oct 2011 21:31:14 -0500 Subject: [PATCH 13/13] Adding csrf token fields to the forgot password calls --- mediagoblin/templates/mediagoblin/auth/change_fp.html | 2 ++ mediagoblin/templates/mediagoblin/auth/forgot_password.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mediagoblin/templates/mediagoblin/auth/change_fp.html b/mediagoblin/templates/mediagoblin/auth/change_fp.html index 4be7e065..53186cec 100644 --- a/mediagoblin/templates/mediagoblin/auth/change_fp.html +++ b/mediagoblin/templates/mediagoblin/auth/change_fp.html @@ -23,6 +23,8 @@
+ {{ csrf_token }} +

{% trans %}Enter your new password{% endtrans %}

diff --git a/mediagoblin/templates/mediagoblin/auth/forgot_password.html b/mediagoblin/templates/mediagoblin/auth/forgot_password.html index 23fa9eb5..b95a4dcb 100644 --- a/mediagoblin/templates/mediagoblin/auth/forgot_password.html +++ b/mediagoblin/templates/mediagoblin/auth/forgot_password.html @@ -23,6 +23,8 @@ + {{ csrf_token }} +

{% trans %}Enter your username or email{% endtrans %}