SQL based tests and refactored Celery setup stuff

- Changed config files of test configs to use SQL
 - Updated celery initialization tools, factored them to be able to
   use the "big instance" application stuff

mediagoblin/gmg_commands/dbupdate.py

@@ -65,14 +65,13 @@ def gather_database_data(media_types):
     return managed_dbdata
 
 
-def dbupdate(args):
+def run_dbupdate(app_config):
     """
     Initialize or migrate the database as specified by the config file.
 
     Will also initialize or migrate all extensions (media types, and
     in the future, plugins)
     """
-    globa_config, app_config = setup_global_and_app_config(args.conf_file)
 
     # Gather information from all media managers / projects
     dbdatas = gather_database_data(app_config['media_types'])
@@ -87,3 +86,8 @@ def dbupdate(args):
     for dbdata in dbdatas:
         migration_manager = dbdata.make_migration_manager(Session())
         migration_manager.init_or_migrate()
+
+
+def dbupdate(args):
+    global_config, app_config = setup_global_and_app_config(args.conf_file)
+    run_dbupdate(app_config)

mediagoblin/init/celery/__init__.py

@@ -17,28 +17,18 @@
 import os
 import sys
 
+from celery import Celery
+
 
 MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task']
 
 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
 
 
-def setup_celery_from_config(app_config, global_config,
+def get_celery_settings_dict(app_config, global_config,
-                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False):
-                             force_celery_always_eager=False,
-                             set_environ=True):
     """
-    Take a mediagoblin app config and try to set up a celery settings
+    Get a celery settings dictionary from reading the config
-    module from this.
-
-    Args:
-    - app_config: the application config section
-    - global_config: the entire ConfigObj loaded config, all sections
-    - settings_module: the module to populate, as a string
-    - force_celery_always_eager: whether or not to force celery into
-      always eager mode; good for development and small installs
-    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
-      settings_module
     """
     if 'celery' in global_config:
         celery_conf = global_config['celery']
@@ -61,6 +51,41 @@ def setup_celery_from_config(app_config, global_config,
         celery_settings['CELERY_ALWAYS_EAGER'] = True
         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
 
+    return celery_settings
+
+
+def setup_celery_app(app_config, global_config,
+                     settings_module=DEFAULT_SETTINGS_MODULE,
+                     force_celery_always_eager=False):
+    """
+    Setup celery without using terrible setup-celery-module hacks.
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+    celery_app = Celery()
+    celery_app.config_from_object(celery_settings)
+
+
+def setup_celery_from_config(app_config, global_config,
+                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False,
+                             set_environ=True):
+    """
+    Take a mediagoblin app config and try to set up a celery settings
+    module from this.
+
+    Args:
+    - app_config: the application config section
+    - global_config: the entire ConfigObj loaded config, all sections
+    - settings_module: the module to populate, as a string
+    - force_celery_always_eager: whether or not to force celery into
+      always eager mode; good for development and small installs
+    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
+      settings_module
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+
     __import__(settings_module)
     this_module = sys.modules[settings_module]
 

mediagoblin/tests/test_mgoblin_app.ini

@@ -2,7 +2,9 @@
 direct_remote_path = /test_static/
 email_sender_address = "notice@mediagoblin.example.org"
 email_debug_mode = true
-db_name = __mediagoblin_tests__
+
+# Use an in-memory database
+sql_engine = "sqlite:///%(here)s/test_user_dev/mediagoblin.db"
 
 # tag parsing
 tags_max_length = 50
@@ -27,3 +29,5 @@ lock_dir = %(here)s/test_user_dev/beaker/cache/lock
 
 [celery]
 CELERY_ALWAYS_EAGER = true
+CELERY_RESULT_DBURI = "sqlite:///%(here)s/test_user_dev/celery.db"
+BROKER_HOST = "sqlite:///%(here)s/test_user_dev/kombu.db"

mediagoblin/tests/tools.py

@@ -26,8 +26,11 @@ from mediagoblin.tools import testing
 from mediagoblin.init.config import read_mediagoblin_config
 from mediagoblin.decorators import _make_safe
 from mediagoblin.db.open import setup_connection_and_db_from_config
+from mediagoblin.db.sql.base import Session
 from mediagoblin.meddleware import BaseMeddleware
 from mediagoblin.auth.lib import bcrypt_gen_password_hash
+from mediagoblin.gmg_commands.dbupdate import run_dbupdate
+from mediagoblin.init.celery import setup_celery_app
 
 
 MEDIAGOBLIN_TEST_DB_NAME = u'__mediagoblin_tests__'
@@ -125,26 +128,19 @@ def get_test_app(dump_old_app=True):
     global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG)
     app_config = global_config['mediagoblin']
 
-    # Wipe database
+    # Run database setup/migrations
-    # @@: For now we're dropping collections, but we could also just
+    run_dbupdate(app_config)
-    # collection.remove() ?
-    connection, db = setup_connection_and_db_from_config(app_config)
-    assert db.name == MEDIAGOBLIN_TEST_DB_NAME
-
-    collections_to_wipe = [
-        collection
-        for collection in db.collection_names()
-        if not collection.startswith('system.')]
-
-    for collection in collections_to_wipe:
-        db.drop_collection(collection)
-
-    # TODO: Drop and recreate indexes
 
     # setup app and return
     test_app = loadapp(
         'config:' + TEST_SERVER_CONFIG)
 
+    Session.rollback()
+    Session.remove()
+
+    # Re-setup celery
+    setup_celery_app(app_config, global_config)
+
     # Insert the TestingMeddleware, which can do some
     # sanity checks on every request/response.
     # Doing it this way is probably not the cleanest way.