Move mediagoblin.celery_setup -> mediagoblin.init.celery

As the first target of the new .init. submodule move
celery_setup there. Quite straight forward, just a lot of
places to change.

mediagoblin/init/celery/__init__.py

@@ -0,0 +1,89 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+
+
+MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media']
+
+DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
+
+
+def setup_celery_from_config(app_config, global_config,
+                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False,
+                             set_environ=True):
+    """
+    Take a mediagoblin app config and try to set up a celery settings
+    module from this.
+
+    Args:
+    - app_config: the application config section
+    - global_config: the entire ConfigObj loaded config, all sections
+    - settings_module: the module to populate, as a string
+    - force_celery_always_eager: whether or not to force celery into
+      always eager mode; good for development and small installs
+    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
+      settings_module
+    """
+    if global_config.has_key('celery'):
+        celery_conf = global_config['celery']
+    else:
+        celery_conf = {}
+    
+    celery_settings = {}
+
+    # set up mongodb stuff
+    celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
+    if not celery_settings.has_key('BROKER_BACKEND'):
+        celery_settings['BROKER_BACKEND'] = 'mongodb'
+
+    celery_mongo_settings = {}
+
+    if app_config.has_key('db_host'):
+        celery_mongo_settings['host'] = app_config['db_host']
+        if celery_settings['BROKER_BACKEND'] == 'mongodb':
+            celery_settings['BROKER_HOST'] = app_config['db_host']
+    if app_config.has_key('db_port'):
+        celery_mongo_settings['port'] = app_config['db_port']
+        if celery_settings['BROKER_BACKEND'] == 'mongodb':
+            celery_settings['BROKER_PORT'] = app_config['db_port']
+    celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
+
+    celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
+
+    # Add anything else
+    for key, value in celery_conf.iteritems():
+        key = key.upper()
+        celery_settings[key] = value
+
+    # add mandatory celery imports
+    celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
+    celery_imports.extend(MANDATORY_CELERY_IMPORTS)
+
+    if force_celery_always_eager:
+        celery_settings['CELERY_ALWAYS_EAGER'] = True
+        celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
+
+    __import__(settings_module)
+    this_module = sys.modules[settings_module]
+
+    for key, value in celery_settings.iteritems():
+        setattr(this_module, key, value)
+    
+    if set_environ:
+        os.environ['CELERY_CONFIG_MODULE'] = settings_module

mediagoblin/init/celery/from_celery.py

@@ -0,0 +1,60 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from mediagoblin import app, mg_globals
+from mediagoblin.init.celery import setup_celery_from_config
+
+
+OUR_MODULENAME = __name__
+
+
+def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME,
+               default_conf_file='mediagoblin.ini'):
+    """
+    Transform this module into a celery config module by reading the
+    mediagoblin config file.  Set the environment variable
+    MEDIAGOBLIN_CONFIG to specify where this config file is.
+
+    By default it defaults to 'mediagoblin.ini'.
+
+    Note that if celery_setup_elsewhere is set in your config file,
+    this simply won't work.
+    """
+    if check_environ_for_conf:
+        mgoblin_conf_file = os.path.abspath(
+            os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file))
+    else:
+        mgoblin_conf_file = default_conf_file
+
+    if not os.path.exists(mgoblin_conf_file):
+        raise IOError(
+            "MEDIAGOBLIN_CONFIG not set or file does not exist")
+        
+    # By setting the environment variable here we should ensure that
+    # this is the module that gets set up.
+    os.environ['CELERY_CONFIG_MODULE'] = module_name
+    app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)
+
+    setup_celery_from_config(
+        mg_globals.app_config, mg_globals.global_config,
+        settings_module=module_name,
+        set_environ=False)
+
+
+if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:
+    setup_self()

mediagoblin/init/celery/from_tests.py

@@ -0,0 +1,33 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from mediagoblin.tests.tools import TEST_APP_CONFIG
+from mediagoblin.init.celery.from_celery import setup_self
+
+
+OUR_MODULENAME = __name__
+CELERY_SETUP = False
+
+
+if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME:
+    if CELERY_SETUP:
+        pass
+    else:
+        setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME,
+                   default_conf_file=TEST_APP_CONFIG)
+        CELERY_SETUP = True