setup_celery_from_config tool. Haven't tried if it works, but looks right...

2011-04-24 18:47:23 -05:00 · 2011-04-24 18:47:23 -05:00 · e231d9e8bb
commit e231d9e8bb
parent df9809c209
1 changed files with 121 additions and 0 deletions
--- a/mediagoblin/celery_setup/init.py
+++ b/mediagoblin/celery_setup/init.py
@ -0,0 +1,121 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from paste.deploy.converters import asbool, asint, aslist
+
+
+KNOWN_CONFIG_BOOLS = [
+    'CELERY_RESULT_PERSISTENT',
+    'CELERY_CREATE_MISSING_QUEUES',
+    'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY',
+    'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS',
+    'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED',
+    'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE',
+    'CELERY_STORE_ERRORS_EVEN_IF_IGNORED',
+    'CELERY_SEND_TASK_ERROR_EMAILS',
+    'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT',
+    'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS',
+    ]
+
+KNOWN_CONFIG_INTS = [
+    'CELERYD_CONCURRENCY',
+    'CELERYD_PREFETCH_MULTIPLIER',
+    'CELERY_AMQP_TASK_RESULT_EXPIRES',
+    'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX',
+    'REDIS_PORT', 'REDIS_DB',
+    'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT',
+    'CELERY_BROKER_CONNECTION_MAX_RETRIES',
+    'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS',
+    'CELERY_DEFAULT_RATE_LIMIT', # ??
+    'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT',
+    'CELERYD_TASK_SOFT_TIME_LIMIT',
+    'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL',
+    ]
+
+KNOWN_CONFIG_FLOATS = [
+    'CELERYD_ETA_SCHEDULER_PRECISION',
+    ]
+
+KNOWN_CONFIG_LISTS = [
+    'CELERY_ROUTES', 'CELERY_IMPORTS',
+    ]
+
+
+## Needs special processing:
+# ADMINS, ???
+# there are a lot more; we should list here or process specially.
+
+
+def asfloat(obj):
+    try:
+        return float(obj)
+    except (TypeError, ValueError), e:
+        raise ValueError(
+            "Bad float value: %r" % obj)
+
+
+def setup_celery_from_config(app_config, global_config):
+    """
+    Take a mediagoblin app config and the global config from a paste
+    factory and try to set up a celery settings module from this.
+    """
+    if asbool(app_config.get('use_celery_environment_var')) == True:
+        # Don't setup celery based on our config file.
+        return
+
+    celery_conf_section = app_config.get('celery_section', 'celery')
+    if global_config.has_key(celery_conf_section):
+        celery_conf = global_config[celery_conf_section]
+    else:
+        celery_conf = {}
+    
+    celery_settings = {}
+
+    # set up mongodb stuff
+    celery_mongo_settings = {}
+    if app_config.has_key('db_host'):
+        celery_mongo_settings['host'] = app_config['db_host']
+        celery_settings['BROKER_HOST'] = app_config['db_host']
+    if app_config.has_key('db_port'):
+        celery_mongo_settings['port'] = asint(app_config['db_port'])
+        celery_settings['BROKER_PORT'] = asint(app_config['db_port'])
+    celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
+
+    celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
+    celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
+
+    # Add anything else
+    for key, value in celery_conf.iteritems():
+        key = key.upper()
+        if key in KNOWN_CONFIG_BOOLS:
+            value = asbool(value)
+        elif value in KNOWN_CONFIG_INTS:
+            value = asint(value)
+        elif value in KNOWN_CONFIG_FLOATS:
+            value = asfloat(value)
+        elif value in KNOWN_CONFIG_LISTS:
+            value = aslist(value)
+        celery_settings[key] = value
+
+    from mediagoblin.celery_setup import dummy_settings_module
+
+    for key, value in celery_settings.iteritems():
+        setattr(dummy_settings_module, key, value)
+
+    os.environ['CELERY_CONFIG_MODULE'] = \
+        'mediagoblin.celery_setup.dummy_settings_module'