Merge remote-tracking branch 'cwebber/celerysql'

* cwebber/celerysql:
  Adjust unit tests to match new celery/kombu sqlalchemy setup
  "database" is not the sqlalchemy kombu transport... should be "sqlalchemy"
  Celery and kombu databases should also be .gitignore'd
  kombu-sqlalchemy a requirement in order for kombu sqlalchemy transport to work
  Move mediagoblin dbs out of user_dev for race condition directory-creation reasons.
  Give kombu its own db.  Responding to Elrond "sqlite will lock all the time!" :)
  Apparently an absolute path is three slashes after sqlite:.  Thx elrond.
  Should be all that's needed to switch celery/kombu settings to sqlalchemy

.gitignore

@@ -16,6 +16,8 @@
 /paste_local.ini
 /mediagoblin_local.ini
 /mediagoblin.db
+/celery.db
+/kombu.db
 /server-log.txt
 
 # Tests

mediagoblin/config_spec.ini

@@ -79,6 +79,14 @@ lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock")
 
 
 [celery]
+# default result stuff
+celery_result_backend = string(default="database")
+celery_result_dburi = string(default="sqlite:///%(here)s/celery.db")
+
+# default kombu stuff
+broker_transport = string(default="sqlalchemy")
+broker_host = string(default="sqlite:///%(here)s/kombu.db")
+
 # known booleans
 celery_result_persistent = boolean()
 celery_create_missing_queues = boolean()

mediagoblin/init/celery/__init__.py

@@ -47,30 +47,13 @@ def setup_celery_from_config(app_config, global_config,
 
     celery_settings = {}
 
-    # set up mongodb stuff
+    # Add all celery settings from config
-    celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
-    if 'BROKER_BACKEND' not in celery_settings:
-        celery_settings['BROKER_BACKEND'] = 'mongodb'
-
-    celery_mongo_settings = {}
-
-    if 'db_host' in app_config:
-        celery_mongo_settings['host'] = app_config['db_host']
-        if celery_settings['BROKER_BACKEND'] == 'mongodb':
-            celery_settings['BROKER_HOST'] = app_config['db_host']
-    if 'db_port' in app_config:
-        celery_mongo_settings['port'] = app_config['db_port']
-        if celery_settings['BROKER_BACKEND'] == 'mongodb':
-            celery_settings['BROKER_PORT'] = app_config['db_port']
-    celery_mongo_settings['database'] = app_config['db_name']
-
-    celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
-
-    # Add anything else
     for key, value in celery_conf.iteritems():
         key = key.upper()
         celery_settings[key] = value
 
+    # TODO: use default result stuff here if it exists
+
     # add mandatory celery imports
     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
     celery_imports.extend(MANDATORY_CELERY_IMPORTS)

mediagoblin/tests/fake_celery_conf_mgdb.ini

@@ -1,14 +0,0 @@
-['mediagoblin']
-db_host = mongodb.example.org
-db_port = 8080
-db_name = captain_lollerskates
-
-['something']
-or = other
-
-['celery']
-some_variable = poolf
-mail_port = 2020
-celeryd_eta_scheduler_precision = 3.1
-celery_result_persistent = false
-celery_imports = baz.bar.foo, import.is.a.this

mediagoblin/tests/test_celery_setup.py

@@ -22,8 +22,6 @@ from mediagoblin.init.config import read_mediagoblin_config
 
 TEST_CELERY_CONF_NOSPECIALDB = pkg_resources.resource_filename(
     'mediagoblin.tests', 'fake_celery_conf.ini')
-TEST_CELERY_CONF_MGSPECIALDB = pkg_resources.resource_filename(
-    'mediagoblin.tests', 'fake_celery_conf_mgdb.ini')
 
 
 def test_setup_celery_from_config():
@@ -51,35 +49,12 @@ def test_setup_celery_from_config():
     assert fake_celery_module.CELERY_RESULT_PERSISTENT is True
     assert fake_celery_module.CELERY_IMPORTS == [
         'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing']
-    assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
+    assert fake_celery_module.CELERY_RESULT_BACKEND == 'database'
-        'database': 'mediagoblin'}
+    assert fake_celery_module.CELERY_RESULT_DBURI == (
-    assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
+        'sqlite:///' +
-    assert fake_celery_module.BROKER_BACKEND == 'mongodb'
+        pkg_resources.resource_filename('mediagoblin.tests', 'celery.db'))
 
-    _wipe_testmodule_clean(fake_celery_module)
+    assert fake_celery_module.BROKER_TRANSPORT == 'sqlalchemy'
-
+    assert fake_celery_module.BROKER_HOST == (
-    global_config, validation_result = read_mediagoblin_config(
+        'sqlite:///' +
-        TEST_CELERY_CONF_MGSPECIALDB)
+        pkg_resources.resource_filename('mediagoblin.tests', 'kombu.db'))
-    app_config = global_config['mediagoblin']
-
-    celery_setup.setup_celery_from_config(
-        app_config, global_config,
-        'mediagoblin.tests.fake_celery_module', set_environ=False)
-    
-    from mediagoblin.tests import fake_celery_module
-    assert fake_celery_module.SOME_VARIABLE == 'poolf'
-    assert fake_celery_module.MAIL_PORT == 2020
-    assert isinstance(fake_celery_module.MAIL_PORT, int)
-    assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1
-    assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float)
-    assert fake_celery_module.CELERY_RESULT_PERSISTENT is False
-    assert fake_celery_module.CELERY_IMPORTS == [
-        'baz.bar.foo', 'import.is.a.this', 'mediagoblin.processing']
-    assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
-        'database': 'captain_lollerskates',
-        'host': 'mongodb.example.org',
-        'port': 8080}
-    assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
-    assert fake_celery_module.BROKER_BACKEND == 'mongodb'
-    assert fake_celery_module.BROKER_HOST == 'mongodb.example.org'
-    assert fake_celery_module.BROKER_PORT == 8080

setup.py

@@ -64,6 +64,7 @@ setup(
         'Markdown',
         'sqlalchemy',
         'sqlalchemy-migrate',
+        'kombu-sqlalchemy',
         ## For now we're expecting that users will install this from
         ## their package managers.
         # 'lxml',