From e629cde0b361be477c5d27cf170fef9eef1d42fe Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 25 Feb 2012 22:39:08 -0600 Subject: [PATCH 1/8] Should be all that's needed to switch celery/kombu settings to sqlalchemy --- mediagoblin/config_spec.ini | 8 ++++++++ mediagoblin/init/celery/__init__.py | 23 +++-------------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 2b4ba2f9..1fd56226 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -79,6 +79,14 @@ lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock") [celery] +# default result stuff +celery_result_backend = string(default="database") +celery_result_dburi = string(default="sqlite://%(here)s/user_dev/celery.db") + +# default kombu stuff +broker_transport = string(default="database") +broker_host = string(default="sqlite://%(here)s/user_dev/celery.db") + # known booleans celery_result_persistent = boolean() celery_create_missing_queues = boolean() diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index fb958909..6dcea239 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -47,30 +47,13 @@ def setup_celery_from_config(app_config, global_config, celery_settings = {} - # set up mongodb stuff - celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' - if 'BROKER_BACKEND' not in celery_settings: - celery_settings['BROKER_BACKEND'] = 'mongodb' - - celery_mongo_settings = {} - - if 'db_host' in app_config: - celery_mongo_settings['host'] = app_config['db_host'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_HOST'] = app_config['db_host'] - if 'db_port' in app_config: - celery_mongo_settings['port'] = app_config['db_port'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_PORT'] = app_config['db_port'] - celery_mongo_settings['database'] = app_config['db_name'] - - celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings - - # Add anything else + # Add all celery settings from config for key, value in celery_conf.iteritems(): key = key.upper() celery_settings[key] = value + # TODO: use default result stuff here if it exists + # add mandatory celery imports celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) celery_imports.extend(MANDATORY_CELERY_IMPORTS) From 200abf856b0c2d6b05e068a2bba9c8afe35e72af Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 07:46:23 -0600 Subject: [PATCH 2/8] Apparently an absolute path is three slashes after sqlite:. Thx elrond. sqlite:///////////////////////////////////whaaaaaa.db --- mediagoblin/config_spec.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 1fd56226..09c5f265 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -81,11 +81,11 @@ lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock") [celery] # default result stuff celery_result_backend = string(default="database") -celery_result_dburi = string(default="sqlite://%(here)s/user_dev/celery.db") +celery_result_dburi = string(default="sqlite:///%(here)s/user_dev/celery.db") # default kombu stuff broker_transport = string(default="database") -broker_host = string(default="sqlite://%(here)s/user_dev/celery.db") +broker_host = string(default="sqlite:///%(here)s/user_dev/celery.db") # known booleans celery_result_persistent = boolean() From b94dfe4cbf6bcf3e46618b3ccf935dee2a69216b Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 07:58:40 -0600 Subject: [PATCH 3/8] Give kombu its own db. Responding to Elrond "sqlite will lock all the time!" :) --- mediagoblin/config_spec.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 09c5f265..1eb50701 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -85,7 +85,7 @@ celery_result_dburi = string(default="sqlite:///%(here)s/user_dev/celery.db") # default kombu stuff broker_transport = string(default="database") -broker_host = string(default="sqlite:///%(here)s/user_dev/celery.db") +broker_host = string(default="sqlite:///%(here)s/user_dev/kombu.db") # known booleans celery_result_persistent = boolean() From b2a7e44c9ea241e78f8e49025b98833808200e32 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 08:20:17 -0600 Subject: [PATCH 4/8] Move mediagoblin dbs out of user_dev for race condition directory-creation reasons. --- mediagoblin/config_spec.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 1eb50701..4088aaa5 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -81,11 +81,11 @@ lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock") [celery] # default result stuff celery_result_backend = string(default="database") -celery_result_dburi = string(default="sqlite:///%(here)s/user_dev/celery.db") +celery_result_dburi = string(default="sqlite:///%(here)s/celery.db") # default kombu stuff broker_transport = string(default="database") -broker_host = string(default="sqlite:///%(here)s/user_dev/kombu.db") +broker_host = string(default="sqlite:///%(here)s/kombu.db") # known booleans celery_result_persistent = boolean() From b594eadfbf90f09f4b2a73b7f89fed2b05ba10f3 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 13:24:49 -0600 Subject: [PATCH 5/8] kombu-sqlalchemy a requirement in order for kombu sqlalchemy transport to work --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1c7caf96..3e382e56 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ setup( 'Markdown', 'sqlalchemy', 'sqlalchemy-migrate', + 'kombu-sqlalchemy', ## For now we're expecting that users will install this from ## their package managers. # 'lxml', From a68e3e8320b43f874bfea7b40e0b1d1342ef631b Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 13:26:44 -0600 Subject: [PATCH 6/8] Celery and kombu databases should also be .gitignore'd --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e3a83822..4c62a766 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ /paste_local.ini /mediagoblin_local.ini /mediagoblin.db +/celery.db +/kombu.db /server-log.txt # Tests From b8e1abf782edd9894974d15ea98637884d22ddf4 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 13:27:09 -0600 Subject: [PATCH 7/8] "database" is not the sqlalchemy kombu transport... should be "sqlalchemy" --- mediagoblin/config_spec.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 4088aaa5..94e90c44 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -84,7 +84,7 @@ celery_result_backend = string(default="database") celery_result_dburi = string(default="sqlite:///%(here)s/celery.db") # default kombu stuff -broker_transport = string(default="database") +broker_transport = string(default="sqlalchemy") broker_host = string(default="sqlite:///%(here)s/kombu.db") # known booleans From a9a63a686e934e3189179dcf01faad39a330a378 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 26 Feb 2012 15:45:58 -0600 Subject: [PATCH 8/8] Adjust unit tests to match new celery/kombu sqlalchemy setup --- mediagoblin/tests/fake_celery_conf_mgdb.ini | 14 ------- mediagoblin/tests/test_celery_setup.py | 41 ++++----------------- 2 files changed, 8 insertions(+), 47 deletions(-) delete mode 100644 mediagoblin/tests/fake_celery_conf_mgdb.ini diff --git a/mediagoblin/tests/fake_celery_conf_mgdb.ini b/mediagoblin/tests/fake_celery_conf_mgdb.ini deleted file mode 100644 index 52671c14..00000000 --- a/mediagoblin/tests/fake_celery_conf_mgdb.ini +++ /dev/null @@ -1,14 +0,0 @@ -['mediagoblin'] -db_host = mongodb.example.org -db_port = 8080 -db_name = captain_lollerskates - -['something'] -or = other - -['celery'] -some_variable = poolf -mail_port = 2020 -celeryd_eta_scheduler_precision = 3.1 -celery_result_persistent = false -celery_imports = baz.bar.foo, import.is.a.this diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py index c9c77821..fd600f56 100644 --- a/mediagoblin/tests/test_celery_setup.py +++ b/mediagoblin/tests/test_celery_setup.py @@ -22,8 +22,6 @@ from mediagoblin.init.config import read_mediagoblin_config TEST_CELERY_CONF_NOSPECIALDB = pkg_resources.resource_filename( 'mediagoblin.tests', 'fake_celery_conf.ini') -TEST_CELERY_CONF_MGSPECIALDB = pkg_resources.resource_filename( - 'mediagoblin.tests', 'fake_celery_conf_mgdb.ini') def test_setup_celery_from_config(): @@ -51,35 +49,12 @@ def test_setup_celery_from_config(): assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing'] - assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { - 'database': 'mediagoblin'} - assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_BACKEND == 'mongodb' + assert fake_celery_module.CELERY_RESULT_BACKEND == 'database' + assert fake_celery_module.CELERY_RESULT_DBURI == ( + 'sqlite:///' + + pkg_resources.resource_filename('mediagoblin.tests', 'celery.db')) - _wipe_testmodule_clean(fake_celery_module) - - global_config, validation_result = read_mediagoblin_config( - TEST_CELERY_CONF_MGSPECIALDB) - app_config = global_config['mediagoblin'] - - celery_setup.setup_celery_from_config( - app_config, global_config, - 'mediagoblin.tests.fake_celery_module', set_environ=False) - - from mediagoblin.tests import fake_celery_module - assert fake_celery_module.SOME_VARIABLE == 'poolf' - assert fake_celery_module.MAIL_PORT == 2020 - assert isinstance(fake_celery_module.MAIL_PORT, int) - assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 - assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) - assert fake_celery_module.CELERY_RESULT_PERSISTENT is False - assert fake_celery_module.CELERY_IMPORTS == [ - 'baz.bar.foo', 'import.is.a.this', 'mediagoblin.processing'] - assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { - 'database': 'captain_lollerskates', - 'host': 'mongodb.example.org', - 'port': 8080} - assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' - assert fake_celery_module.BROKER_PORT == 8080 + assert fake_celery_module.BROKER_TRANSPORT == 'sqlalchemy' + assert fake_celery_module.BROKER_HOST == ( + 'sqlite:///' + + pkg_resources.resource_filename('mediagoblin.tests', 'kombu.db'))