Proper webtest infrastructure... seems to be about right anyway :)

mediagoblin/app.py

@@ -139,12 +139,13 @@ def paste_app_factory(global_config, **app_config):
         raise ImproperlyConfigured(
             "One of direct_remote_path or direct_remote_paths must be provided")
 
-    if asbool(os.environ.get('CELERY_ALWAYS_EAGER')):
+    if not asbool(app_config.get('celery_setup_elsewhere')):
-        setup_celery_from_config(
+        if asbool(os.environ.get('CELERY_ALWAYS_EAGER')):
-            app_config, global_config,
+            setup_celery_from_config(
-            force_celery_always_eager=True)
+                app_config, global_config,
-    else:
+                force_celery_always_eager=True)
-        setup_celery_from_config(app_config, global_config)
+        else:
+            setup_celery_from_config(app_config, global_config)
 
     mgoblin_app = MediaGoblinApp(
         connection, db,

mediagoblin/celery_setup/from_tests.py

@@ -0,0 +1,43 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from mediagoblin.tests.tools import TEST_APP_CONFIG
+from mediagoblin import util
+from mediagoblin.celery_setup import setup_celery_from_config
+from mediagoblin.globals import setup_globals
+
+
+OUR_MODULENAME = 'mediagoblin.celery_setup.from_tests'
+
+
+def setup_self(setup_globals_func=setup_globals):
+    """
+    Set up celery for testing's sake, which just needs to set up
+    celery and celery only.
+    """
+    mgoblin_conf = util.read_config_file(TEST_APP_CONFIG)
+    mgoblin_section = mgoblin_conf['app:mediagoblin']
+
+    setup_celery_from_config(
+        mgoblin_section, mgoblin_conf,
+        settings_module=OUR_MODULENAME,
+        set_environ=False)
+
+
+if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME:
+    setup_self()

mediagoblin/tests/mgoblin_test_app.ini

@@ -0,0 +1,46 @@
+[DEFAULT]
+debug = true
+
+[composite:main]
+use = egg:Paste#urlmap
+/ = mediagoblin
+/mgoblin_media/ = publicstore_serve
+/mgoblin_static/ = mediagoblin_static
+
+[app:mediagoblin]
+use = egg:mediagoblin#app
+filter-with = beaker
+queuestore_base_dir = %(here)s/test_user_dev/media/queue
+publicstore_base_dir = %(here)s/test_user_dev/media/public
+publicstore_base_url = /mgoblin_media/
+direct_remote_path = /mgoblin_static/
+email_sender_address = "notice@mediagoblin.example.org"
+email_debug_mode = true
+db_name = __mediagoblin_tests__
+# Celery shouldn't be set up by the paste app factory as it's set up
+# elsewhere
+celery_setup_elsewhere = true
+
+[app:publicstore_serve]
+use = egg:Paste#static
+document_root = %(here)s/user_dev/media/public
+
+[app:mediagoblin_static]
+use = egg:Paste#static
+document_root = %(here)s/mediagoblin/static/
+
+[filter:beaker]
+use = egg:Beaker#beaker_session
+cache_dir = %(here)s/test_user_dev/beaker
+beaker.session.key = mediagoblin
+# beaker.session.secret = somesupersecret
+beaker.session.data_dir = %(here)s/test_user_dev/beaker/sessions/data
+beaker.session.lock_dir = %(here)s/test_user_dev/beaker/sessions/lock
+
+[celery]
+celery_always_eager = true
+
+[server:main]
+use = egg:Paste#http
+host = 127.0.0.1
+port = 6543

mediagoblin/tests/tools.py

@@ -0,0 +1,94 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import pkg_resources
+import os, shutil
+
+from paste.deploy import appconfig
+from webtest import TestApp
+
+from mediagoblin import app
+from mediagoblin.db.open import setup_connection_and_db_from_config
+
+
+MEDIAGOBLIN_TEST_DB_NAME = '__mediagoblinunittests__'
+TEST_APP_CONFIG = pkg_resources.resource_filename(
+    'mediagoblin.tests', 'mgoblin_test_app.ini')
+TEST_USER_DEV = pkg_resources.resource_filename(
+    'mediagoblin.tests', 'test_user_dev')
+MGOBLIN_APP = None
+
+USER_DEV_DIRECTORIES_TO_SETUP = [
+    'media/public', 'media/queue',
+    'beaker/sessions/data', 'beaker/sessions/lock']
+
+
+class BadCeleryEnviron(Exception): pass
+
+
+def get_test_app(dump_old_app=True):
+    if not os.environ.get('CELERY_CONFIG_MODULE') == \
+            'mediagoblin.celery_setup.from_tests':
+        raise BadCeleryEnviron(
+            u"Sorry, you *absolutely* must run nosetests with the\n"
+            u"mediagoblin.celery_setup.from_tests module.  Like so:\n"
+            u"$ CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests")
+
+    # Just return the old app if that exists and it's okay to set up
+    # and return
+    if MGOBLIN_APP and not dump_old_app:
+        return MGOBLIN_APP
+
+    # Remove and reinstall user_dev directories
+    if os.path.exists(TEST_USER_DEV):
+        shutil.rmtree(TEST_USER_DEV)
+
+    for directory in USER_DEV_DIRECTORIES_TO_SETUP:
+        full_dir = os.path.join(TEST_USER_DEV, directory)
+        os.makedirs(full_dir)
+
+    # Get app config
+    config = appconfig(
+        'config:' + os.path.basename(TEST_APP_CONFIG),
+        relative_to=os.path.dirname(TEST_APP_CONFIG),
+        name='mediagoblin')
+
+    # Wipe database
+    # @@: For now we're dropping collections, but we could also just
+    # collection.remove() ?
+    connection, db = setup_connection_and_db_from_config(
+        config.local_conf)
+
+    collections_to_wipe = [
+        collection
+        for collection in db.collection_names()
+        if not collection.startswith('system.')]
+
+    for collection in collections_to_wipe:
+        db.drop_collection(collection)
+
+    # Don't need these anymore...
+    del(connection)
+    del(db)
+
+    # TODO: Drop and recreate indexes
+
+    # setup app and return
+    test_app = app.paste_app_factory(
+        config.global_conf, **config.local_conf)
+
+    return TestApp(test_app)

mediagoblin/util.py

@@ -18,18 +18,21 @@ from email.MIMEText import MIMEText
 import gettext
 import pkg_resources
 import smtplib
+import os
 import sys
 import re
-import jinja2
-from mediagoblin.db.util import ObjectId
-import translitcodec
-
-from mediagoblin import globals as mgoblin_globals
-
 import urllib
 from math import ceil
 import copy
 
+import jinja2
+import translitcodec
+from paste.deploy.loadwsgi import NicerConfigParser
+
+from mediagoblin import globals as mgoblin_globals
+from mediagoblin.db.util import ObjectId
+
+
 TESTS_ENABLED = False
 def _activate_testing():
     """
@@ -278,6 +281,28 @@ def get_locale_from_request(request):
     return locale_to_lower_upper(target_lang)
 
 
+def read_config_file(conf_file):
+    """
+    Read a paste deploy style config file and process it.
+    """
+    if not os.path.exists(conf_file):
+        raise IOError(
+            "MEDIAGOBLIN_CONFIG not set or file does not exist")
+
+    parser = NicerConfigParser(conf_file)
+    parser.read(conf_file)
+    parser._defaults.setdefault(
+        'here', os.path.dirname(os.path.abspath(conf_file)))
+    parser._defaults.setdefault(
+        '__file__', os.path.abspath(conf_file))
+
+    mgoblin_conf = dict(
+        [(section_name, dict(parser.items(section_name)))
+         for section_name in parser.sections()])
+
+    return mgoblin_conf
+
+
 def setup_gettext(locale):
     """
     Setup the gettext instance based on this locale

setup.py

@@ -41,6 +41,7 @@ setup(
         'Babel',
         'translitcodec',
         'argparse',
+        'webtest',
         ],
     test_suite='nose.collector',