Merge branch 'master' into f411_new_migrations

Conflicts:
	mediagoblin/db/open.py

docs/deploymenthowto.rst

@@ -10,4 +10,7 @@ Step 2: ?
 
 Step 3: Write the deployment guide and profit!
 
-But seriously, this is a stub since we're not quite there, yet.
+But seriously, this is a stub since we're not quite there (yet) but if
+you want to see where we are now, you can try to run the latest
+development version by following the instructions at
+:ref:`hacking-howto`.

docs/hackinghowto.rst

@@ -60,7 +60,7 @@ requirements::
 On Fedora::
 
     yum install mongodb-server python-paste-deploy python-paste-script \
-         git-core python python-devel
+         git-core python python-devel python-lxml
 
 .. YouCanHelp::
 

mediagoblin/app.py

@@ -20,19 +20,13 @@ import urllib
 import routes
 from webob import Request, exc
 
-from mediagoblin import routing, util, storage, staticdirect
+from mediagoblin import routing, util, storage
-from mediagoblin.init.config import (
-    read_mediagoblin_config, generate_validation_report)
 from mediagoblin.db.open import setup_connection_and_db_from_config
 from mediagoblin.db.util import MigrationManager
 from mediagoblin.mg_globals import setup_globals
 from mediagoblin.init.celery import setup_celery_from_config
-from mediagoblin.init import get_jinja_loader
+from mediagoblin.init import get_jinja_loader, get_staticdirector, \
-from mediagoblin.workbench import WorkbenchManager
+    setup_global_and_app_config, setup_workbench
-
-
-class Error(Exception): pass
-class ImproperlyConfigured(Error): pass
 
 
 class MediaGoblinApp(object):
@@ -56,13 +50,7 @@ class MediaGoblinApp(object):
         ##############
 
         # Open and setup the config
-        global_config, validation_result = read_mediagoblin_config(config_path)
+        global_config, app_config = setup_global_and_app_config(config_path)
-        app_config = global_config['mediagoblin']
-        # report errors if necessary
-        validation_report = generate_validation_report(
-            global_config, validation_result)
-        if validation_report:
-            raise ImproperlyConfigured(validation_report)
 
         ##########################################
         # Setup other connections / useful objects
@@ -96,19 +84,7 @@ class MediaGoblinApp(object):
         self.routing = routing.get_mapper()
 
         # set up staticdirector tool
-        if app_config.has_key('direct_remote_path'):
+        self.staticdirector = get_staticdirector(app_config)
-            self.staticdirector = staticdirect.RemoteStaticDirect(
-                app_config['direct_remote_path'].strip())
-        elif app_config.has_key('direct_remote_paths'):
-            direct_remote_path_lines = app_config[
-                'direct_remote_paths'].strip().splitlines()
-            self.staticdirector = staticdirect.MultiRemoteStaticDirect(
-                dict([line.strip().split(' ', 1)
-                      for line in direct_remote_path_lines]))
-        else:
-            raise ImproperlyConfigured(
-                "One of direct_remote_path or "
-                "direct_remote_paths must be provided")
 
         # Setup celery, if appropriate
         if setup_celery and not app_config.get('celery_setup_elsewhere'):
@@ -128,21 +104,15 @@ class MediaGoblinApp(object):
         #######################################################
 
         setup_globals(
-            app_config=app_config,
-            global_config=global_config,
-
-            # TODO: No need to set these two up as globals, we could
-            # just read them out of mg_globals.app_config
-            email_sender_address=app_config['email_sender_address'],
-            email_debug_mode=app_config['email_debug_mode'],
-
-            # Actual, useful to everyone objects
             app=self,
             db_connection=self.connection,
             database=self.db,
             public_store=self.public_store,
-            queue_store=self.queue_store,
+            queue_store=self.queue_store)
-            workbench_manager=WorkbenchManager(app_config['workbench_path']))
+
+        # Workbench *currently* only used by celery, so this only
+        # matters in always eager mode :)
+        setup_workbench()
 
     def __call__(self, environ, start_response):
         request = Request(environ)

mediagoblin/auth/lib.py

@@ -112,7 +112,7 @@ def send_verification_email(user, request):
 
     # TODO: There is no error handling in place
     send_email(
-        mg_globals.email_sender_address,
+        mg_globals.app_config['email_sender_address'],
         [user['email']],
         # TODO
         # Due to the distributed nature of GNU MediaGoblin, we should

mediagoblin/config_spec.ini

@@ -1,7 +1,7 @@
 [mediagoblin]
 # database stuff
 db_host = string()
-db_name = string()
+db_name = string(default="mediagoblin")
 db_port = integer()
 
 # 

mediagoblin/db/indexes.py

@@ -45,11 +45,13 @@ REQUIRED READING:
 To remove deprecated indexes
 ----------------------------
 
-Removing deprecated indexes is easier, just do:
+Removing deprecated indexes is the same, just move the index into the
+deprecated indexes mapping.
 
-INACTIVE_INDEXES = {
+DEPRECATED_INDEXES = {
-    'collection_name': [
+    'collection_name': {
-        'deprecated_index_identifier1', 'deprecated_index_identifier2']}
+        'deprecated_index_identifier1': {
+            'index': [index_foo_goes_here]}}
         
 ... etc.
 

mediagoblin/db/open.py

@@ -38,6 +38,7 @@ def connect_database_from_config(app_config, use_pymongo=False):
             app_config.get('db_host'), port)
     return connection
 
+
 def setup_connection_and_db_from_config(app_config, use_pymongo=False):
     """
     Setup connection and database from config.
@@ -45,7 +46,7 @@ def setup_connection_and_db_from_config(app_config, use_pymongo=False):
     Optionally use pymongo instead of mongokit.
     """
     connection = connect_database_from_config(app_config, use_pymongo)
-    database_path = app_config.get('db_name', 'mediagoblin')
+    database_path = app_config['db_name']
     db = connection[database_path]
 
     if not use_pymongo:

mediagoblin/db/util.py

@@ -86,18 +86,25 @@ def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES):
     Args:
      - database: pymongo or mongokit database instance.
      - deprecated_indexes: the indexes to deprecate in the pattern of:
-       {'collection': ['index_identifier1', 'index_identifier2']}
+       {'collection_name': {
+            'identifier': {
+                'index': [index_foo_goes_here],
+                'unique': True}}
+
+       (... although we really only need the 'identifier' here, as the
+       rest of the information isn't used in this case.  But it's kept
+       around so we can remember what it was)
 
     Returns:
       A list of indexes removed in form ('collection', 'index_name')
     """
     indexes_removed = []
 
-    for collection_name, index_names in deprecated_indexes.iteritems():
+    for collection_name, indexes in deprecated_indexes.iteritems():
         collection = database[collection_name]
         collection_indexes = collection.index_information().keys()
 
-        for index_name in index_names:
+        for index_name, index_data in indexes.iteritems():
             if index_name in collection_indexes:
                 collection.drop_index(index_name)
 

mediagoblin/init/__init__.py

@@ -15,8 +15,33 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import jinja2
+from mediagoblin import staticdirect
+from mediagoblin.init.config import (
+    read_mediagoblin_config, generate_validation_report)
+from mediagoblin import mg_globals
+from mediagoblin.mg_globals import setup_globals
+from mediagoblin.workbench import WorkbenchManager
 
 
+class Error(Exception): pass
+class ImproperlyConfigured(Error): pass
+
+
+def setup_global_and_app_config(config_path):
+    global_config, validation_result = read_mediagoblin_config(config_path)
+    app_config = global_config['mediagoblin']
+    # report errors if necessary
+    validation_report = generate_validation_report(
+        global_config, validation_result)
+    if validation_report:
+        raise ImproperlyConfigured(validation_report)
+
+    setup_globals(
+        app_config=app_config,
+        global_config=global_config)
+
+    return global_config, app_config
+
 def get_jinja_loader(user_template_path=None):
     """
     Set up the Jinja template loaders, possibly allowing for user
@@ -31,3 +56,27 @@ def get_jinja_loader(user_template_path=None):
              jinja2.PackageLoader('mediagoblin', 'templates')])
     else:
         return jinja2.PackageLoader('mediagoblin', 'templates')
+
+
+def get_staticdirector(app_config):
+    if app_config.has_key('direct_remote_path'):
+        return staticdirect.RemoteStaticDirect(
+            app_config['direct_remote_path'].strip())
+    elif app_config.has_key('direct_remote_paths'):
+        direct_remote_path_lines = app_config[
+            'direct_remote_paths'].strip().splitlines()
+        return staticdirect.MultiRemoteStaticDirect(
+            dict([line.strip().split(' ', 1)
+                  for line in direct_remote_path_lines]))
+    else:
+        raise ImproperlyConfigured(
+            "One of direct_remote_path or "
+            "direct_remote_paths must be provided")
+
+
+def setup_workbench():
+    app_config = mg_globals.app_config
+
+    workbench_manager = WorkbenchManager(app_config['workbench_path'])
+
+    setup_globals(workbench_manager = workbench_manager)

mediagoblin/init/celery/__init__.py

@@ -62,7 +62,7 @@ def setup_celery_from_config(app_config, global_config,
         celery_mongo_settings['port'] = app_config['db_port']
         if celery_settings['BROKER_BACKEND'] == 'mongodb':
             celery_settings['BROKER_PORT'] = app_config['db_port']
-    celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
+    celery_mongo_settings['database'] = app_config['db_name']
 
     celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
 

mediagoblin/mg_globals.py

@@ -20,12 +20,6 @@ database = None
 public_store = None
 queue_store = None
 
-# Dump mail to stdout instead of sending it:
-email_debug_mode = False
-
-# Address for sending out mails
-email_sender_address = None
-
 # A WorkBenchManager
 workbench_manager = None
 

mediagoblin/util.py

@@ -265,9 +265,9 @@ def send_email(from_addr, to_addrs, subject, message_body):
      - message_body: email body text
     """
     # TODO: make a mock mhost if testing is enabled
-    if TESTS_ENABLED or mg_globals.email_debug_mode:
+    if TESTS_ENABLED or mg_globals.app_config['email_debug_mode']:
         mhost = FakeMhost()
-    elif not mg_globals.email_debug_mode:
+    elif not mg_globals.app_config['email_debug_mode']:
         mhost = smtplib.SMTP()
 
     mhost.connect()
@@ -280,7 +280,7 @@ def send_email(from_addr, to_addrs, subject, message_body):
     if TESTS_ENABLED:
         EMAIL_TEST_INBOX.append(message)
 
-    if getattr(mg_globals, 'email_debug_mode', False):
+    if mg_globals.app_config['email_debug_mode']:
         print u"===== Email ====="
         print u"From address: %s" % message['From']
         print u"To addresses: %s" % message['To']