Merge remote-tracking branch 'refs/remotes/elrond/sql/final'

Conflicts:
	mediagoblin/tests/test_submission.py

Also, WHOO SQL SWITCHOVER PARTY!

        ASCII DANCE PARTY

    /_o_/    \ /   \o_    o
     (       _|_    )    //)
     /\     / o \  /|     /|

      *BMCH BMCH BMCH BMCH*
%
 /_o_/   HHHYAAaaaaa
  /_
 / /
%
AAAAAHAHAHAHAHHHAAHA
       ,,          .------
      o_o ;;       /\\     \    $  __
      '\/ ||      // \\     #     /_/
        \//      // //\\     \
         )             \\     \   %
        \\              \\_____\
        | )            //-------
       /_/_           //     //

       SWITCH YOUR DATABASE
     FLIP A FUKKEN BOOLEAN
%
             __________
          .-'          '-.
        .'                '.
      .'  _--_        _--_  '.
     /   /  (_).     /  (_).  \
    .   |      |    |      |   .
    |    ._____,     ._____,   |
    |   ____________________   |
    |    |                 |   |
    '    \                 /   '
     \    '.        .----./   /
      \     '._    /     /   /
       '.      '--------'  .'
         '._            _.'
            '----------'

.gitignore

@@ -20,7 +20,6 @@
 /celery.db
 /kombu.db
 /server-log.txt
-/mediagoblin/db/sql_switch.py
 
 # Tests
 /mediagoblin/tests/user_dev/

mediagoblin/db/sql_switch.py

@@ -0,0 +1 @@
+use_sql = True

mediagoblin/gmg_commands/dbupdate.py

@@ -65,14 +65,13 @@ def gather_database_data(media_types):
     return managed_dbdata
 
 
-def dbupdate(args):
+def run_dbupdate(app_config):
     """
     Initialize or migrate the database as specified by the config file.
 
     Will also initialize or migrate all extensions (media types, and
     in the future, plugins)
     """
-    globa_config, app_config = setup_global_and_app_config(args.conf_file)
 
     # Gather information from all media managers / projects
     dbdatas = gather_database_data(app_config['media_types'])
@@ -87,3 +86,8 @@ def dbupdate(args):
     for dbdata in dbdatas:
         migration_manager = dbdata.make_migration_manager(Session())
         migration_manager.init_or_migrate()
+
+
+def dbupdate(args):
+    global_config, app_config = setup_global_and_app_config(args.conf_file)
+    run_dbupdate(app_config)

mediagoblin/init/celery/__init__.py

@@ -17,28 +17,18 @@
 import os
 import sys
 
+from celery import Celery
+
 
 MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task']
 
 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
 
 
-def setup_celery_from_config(app_config, global_config,
+def get_celery_settings_dict(app_config, global_config,
-                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False):
-                             force_celery_always_eager=False,
-                             set_environ=True):
     """
-    Take a mediagoblin app config and try to set up a celery settings
+    Get a celery settings dictionary from reading the config
-    module from this.
-
-    Args:
-    - app_config: the application config section
-    - global_config: the entire ConfigObj loaded config, all sections
-    - settings_module: the module to populate, as a string
-    - force_celery_always_eager: whether or not to force celery into
-      always eager mode; good for development and small installs
-    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
-      settings_module
     """
     if 'celery' in global_config:
         celery_conf = global_config['celery']
@@ -61,6 +51,41 @@ def setup_celery_from_config(app_config, global_config,
         celery_settings['CELERY_ALWAYS_EAGER'] = True
         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
 
+    return celery_settings
+
+
+def setup_celery_app(app_config, global_config,
+                     settings_module=DEFAULT_SETTINGS_MODULE,
+                     force_celery_always_eager=False):
+    """
+    Setup celery without using terrible setup-celery-module hacks.
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+    celery_app = Celery()
+    celery_app.config_from_object(celery_settings)
+
+
+def setup_celery_from_config(app_config, global_config,
+                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False,
+                             set_environ=True):
+    """
+    Take a mediagoblin app config and try to set up a celery settings
+    module from this.
+
+    Args:
+    - app_config: the application config section
+    - global_config: the entire ConfigObj loaded config, all sections
+    - settings_module: the module to populate, as a string
+    - force_celery_always_eager: whether or not to force celery into
+      always eager mode; good for development and small installs
+    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
+      settings_module
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+
     __import__(settings_module)
     this_module = sys.modules[settings_module]
 

mediagoblin/tests/__init__.py

@@ -14,10 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from mediagoblin import mg_globals
+import os
+import shutil
 
+from mediagoblin import mg_globals
 from mediagoblin.tests.tools import (
-    MEDIAGOBLIN_TEST_DB_NAME, suicide_if_bad_celery_environ)
+    TEST_USER_DEV, suicide_if_bad_celery_environ)
 
 
 def setup_package():
@@ -25,8 +27,6 @@ def setup_package():
 
 
 def teardown_package():
-    if ((mg_globals.db_connection
+    # Remove and reinstall user_dev directories
-         and mg_globals.database.name == MEDIAGOBLIN_TEST_DB_NAME)):
+    if os.path.exists(TEST_USER_DEV):
-            print "Killing db ..."
+        shutil.rmtree(TEST_USER_DEV)
-            mg_globals.db_connection.drop_database(MEDIAGOBLIN_TEST_DB_NAME)
-            print "... done"

mediagoblin/tests/test_auth.py

@@ -269,6 +269,7 @@ def test_register_views(test_app):
 
     ## Try using an expired token to change password, shouldn't work
     template.clear_test_template_context()
+    new_user = mg_globals.database.User.find_one({'username': 'happygirl'})
     real_token_expiration = new_user.fp_token_expire
     new_user.fp_token_expire = datetime.datetime.now()
     new_user.save()

mediagoblin/tests/test_mgoblin_app.ini

@@ -2,7 +2,9 @@
 direct_remote_path = /test_static/
 email_sender_address = "notice@mediagoblin.example.org"
 email_debug_mode = true
-db_name = __mediagoblin_tests__
+
+# TODO: Switch to using an in-memory database
+sql_engine = "sqlite:///%(here)s/test_user_dev/mediagoblin.db"
 
 # tag parsing
 tags_max_length = 50
@@ -27,3 +29,5 @@ lock_dir = %(here)s/test_user_dev/beaker/cache/lock
 
 [celery]
 CELERY_ALWAYS_EAGER = true
+CELERY_RESULT_DBURI = "sqlite:///%(here)s/test_user_dev/celery.db"
+BROKER_HOST = "sqlite:///%(here)s/test_user_dev/kombu.db"

mediagoblin/tests/test_submission.py

@@ -81,8 +81,8 @@ class TestSubmission:
     def upload_data(self, filename):
         return {'upload_files': [('file', filename)]}
 
-    def check_comments(self, request, media, count):
+    def check_comments(self, request, media_id, count):
-        comments = request.db.MediaComment.find({'media_entry': media._id})
+        comments = request.db.MediaComment.find({'media_entry': media_id})
         assert_equal(count, len(list(comments)))
 
     def test_missing_fields(self):
@@ -134,9 +134,11 @@ class TestSubmission:
                                          *REQUEST_CONTEXT, do_follow=True,
                                          **self.upload_data(GOOD_JPG))
         media = self.check_media(request, {'title': 'Balanced Goblin'}, 1)
-        assert_equal(media.tags,
+        assert media.tags[0]['name'] == u'yin'
-                     [{'name': u'yin', 'slug': u'yin'},
+        assert media.tags[0]['slug'] == u'yin'
-                      {'name': u'yang', 'slug': u'yang'}])
+
+        assert media.tags[1]['name'] == u'yang'
+        assert media.tags[1]['slug'] == u'yang'
 
         # Test tags that are too long
         # ---------------
@@ -154,31 +156,33 @@ class TestSubmission:
                                          *REQUEST_CONTEXT, do_follow=True,
                                          **self.upload_data(GOOD_JPG))
         media = self.check_media(request, {'title': 'Balanced Goblin'}, 1)
+        media_id = media.id
 
         # Add a comment, so we can test for its deletion later.
-        self.check_comments(request, media, 0)
+        self.check_comments(request, media_id, 0)
         comment_url = request.urlgen(
             'mediagoblin.user_pages.media_post_comment',
-            user=self.test_user.username, media=media._id)
+            user=self.test_user.username, media=media_id)
         response = self.do_post({'comment_content': 'i love this test'},
                                 url=comment_url, do_follow=True)[0]
-        self.check_comments(request, media, 1)
+        self.check_comments(request, media_id, 1)
 
         # Do not confirm deletion
         # ---------------------------------------------------
         delete_url = request.urlgen(
             'mediagoblin.user_pages.media_confirm_delete',
-            user=self.test_user.username, media=media._id)
+            user=self.test_user.username, media=media_id)
         # Empty data means don't confirm
         response = self.do_post({}, do_follow=True, url=delete_url)[0]
         media = self.check_media(request, {'title': 'Balanced Goblin'}, 1)
+        media_id = media.id
 
         # Confirm deletion
         # ---------------------------------------------------
         response, request = self.do_post({'confirm': 'y'}, *REQUEST_CONTEXT,
                                          do_follow=True, url=delete_url)
-        self.check_media(request, {'_id': media._id}, 0)
+        self.check_media(request, {'_id': media_id}, 0)
-        self.check_comments(request, media, 0)
+        self.check_comments(request, media_id, 0)
 
     def test_evil_file(self):
         # Test non-suppoerted file with non-supported extension
@@ -246,7 +250,7 @@ class TestSubmission:
             filename = resource_filename(
                 'mediagoblin.tests',
                 os.path.join('test_user_dev/media/public',
-                             *media['media_files'].get(key, [])))
+                             *media.media_files.get(key, [])))
             assert_true(filename.endswith('_' + basename))
             # Is it smaller than the last processed image we looked at?
             size = os.stat(filename).st_size

mediagoblin/tests/tools.py

@@ -26,8 +26,11 @@ from mediagoblin.tools import testing
 from mediagoblin.init.config import read_mediagoblin_config
 from mediagoblin.decorators import _make_safe
 from mediagoblin.db.open import setup_connection_and_db_from_config
+from mediagoblin.db.sql.base import Session
 from mediagoblin.meddleware import BaseMeddleware
 from mediagoblin.auth.lib import bcrypt_gen_password_hash
+from mediagoblin.gmg_commands.dbupdate import run_dbupdate
+from mediagoblin.init.celery import setup_celery_app
 
 
 MEDIAGOBLIN_TEST_DB_NAME = u'__mediagoblin_tests__'
@@ -125,26 +128,19 @@ def get_test_app(dump_old_app=True):
     global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG)
     app_config = global_config['mediagoblin']
 
-    # Wipe database
+    # Run database setup/migrations
-    # @@: For now we're dropping collections, but we could also just
+    run_dbupdate(app_config)
-    # collection.remove() ?
-    connection, db = setup_connection_and_db_from_config(app_config)
-    assert db.name == MEDIAGOBLIN_TEST_DB_NAME
-
-    collections_to_wipe = [
-        collection
-        for collection in db.collection_names()
-        if not collection.startswith('system.')]
-
-    for collection in collections_to_wipe:
-        db.drop_collection(collection)
-
-    # TODO: Drop and recreate indexes
 
     # setup app and return
     test_app = loadapp(
         'config:' + TEST_SERVER_CONFIG)
 
+    Session.rollback()
+    Session.remove()
+
+    # Re-setup celery
+    setup_celery_app(app_config, global_config)
+
     # Insert the TestingMeddleware, which can do some
     # sanity checks on every request/response.
     # Doing it this way is probably not the cleanest way.
@@ -216,4 +212,11 @@ def fixture_add_user(username = u'chris', password = 'toast',
 
     test_user.save()
 
+    # Reload
+    test_user = mg_globals.database.User.find_one({'username': username})
+
+    # ... and detach from session:
+    from mediagoblin.db.sql.base import Session
+    Session.expunge(test_user)
+
     return test_user