Merge remote-tracking branch 'gitorious/master'

mediagoblin/db/indexes.py

@@ -0,0 +1,118 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Indexes for the local database.
+
+To add new indexes
+------------------
+
+Indexes are recorded in the following format:
+
+ACTIVE_INDEXES = {
+    'collection_name': {
+        'identifier': {  # key identifier used for possibly deprecating later
+            'index': [index_foo_goes_here]}}
+
+... and anything else being parameters to the create_index function
+(including unique=True, etc)
+
+Current indexes must be registered in ACTIVE_INDEXES... deprecated
+indexes should be marked in DEPRECATED_INDEXES.
+
+Remember, ordering of compound indexes MATTERS.  Read below for more.
+
+REQUIRED READING:
+ - http://kylebanker.com/blog/2010/09/21/the-joy-of-mongodb-indexes/
+
+ - http://www.mongodb.org/display/DOCS/Indexes
+ - http://www.mongodb.org/display/DOCS/Indexing+Advice+and+FAQ
+
+
+To remove deprecated indexes
+----------------------------
+
+Removing deprecated indexes is easier, just do:
+
+INACTIVE_INDEXES = {
+    'collection_name': [
+        'deprecated_index_identifier1', 'deprecated_index_identifier2']}
+        
+... etc.
+
+If an index has been deprecated that identifier should NEVER BE USED
+AGAIN.  Eg, if you previously had 'awesomepants_unique', you shouldn't
+use 'awesomepants_unique' again, you should create a totally new name
+or at worst use 'awesomepants_unique2'.
+"""
+
+from pymongo import ASCENDING, DESCENDING
+
+
+################
+# Active indexes
+################
+ACTIVE_INDEXES = {}
+
+# MediaEntry indexes
+# ------------------
+
+MEDIAENTRY_INDEXES = {
+    'uploader_slug_unique': {
+        # Matching an object to an uploader + slug.
+        # MediaEntries are unique on these two combined, eg:
+        #   /u/${myuser}/m/${myslugname}/
+        'index': [('uploader', ASCENDING),
+                  ('slug', ASCENDING)],
+        'unique': True},
+
+    'created': {
+        # A global index for all media entries created, in descending
+        # order.  This is used for the site's frontpage.
+        'index': [('created', DESCENDING)]},
+
+    'uploader_created': {
+        # Indexing on uploaders and when media entries are created.
+        # Used for showing a user gallery, etc.
+        'index': [('uploader', ASCENDING),
+                  ('created', DESCENDING)]}}
+
+
+ACTIVE_INDEXES['media_entries'] = MEDIAENTRY_INDEXES
+
+
+# User indexes
+# ------------
+
+USER_INDEXES = {
+    'username_unique': {
+        # Index usernames, and make sure they're unique.
+        # ... I guess we might need to adjust this once we're federated :)
+        'index': 'username',
+        'unique': True},
+    'created': {
+        # All most recently created users
+        'index': 'created'}}
+
+
+ACTIVE_INDEXES['users'] = USER_INDEXES
+
+
+####################
+# Deprecated indexes
+####################
+
+DEPRECATED_INDEXES = {}

mediagoblin/db/models.py

@@ -108,11 +108,6 @@ class MediaEntry(Document):
 
     migration_handler = migrations.MediaEntryMigration
 
-    indexes = [
-        # Referene uniqueness of slugs by uploader
-        {'fields': ['uploader', 'slug'],
-         'unique': True}]
-
     def main_mediafile(self):
         pass
 

mediagoblin/db/util.py

@@ -14,8 +14,88 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+"""
+Utilities for database operations.
+
+Some note on migration and indexing tools:
+
+We store information about what the state of the database is in the
+'mediagoblin' document of the 'app_metadata' collection.  Keys in that
+document relevant to here:
+
+ - 'migration_number': The integer representing the current state of
+   the migrations
+"""
+
+import copy
 
 # Imports that other modules might use
 from pymongo import DESCENDING
 from pymongo.errors import InvalidId
 from mongokit import ObjectId
+
+from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES
+
+
+def add_new_indexes(database, active_indexes=ACTIVE_INDEXES):
+    """
+    Add any new indexes to the database.
+
+    Args:
+     - database: pymongo or mongokit database instance.
+     - active_indexes: indexes to possibly add in the pattern of:
+       {'collection_name': {
+            'identifier': {
+                'index': [index_foo_goes_here],
+                'unique': True}}
+       where 'index' is the index to add and all other options are
+       arguments for collection.create_index.
+
+    Returns:
+      A list of indexes added in form ('collection', 'index_name')
+    """
+    indexes_added = []
+
+    for collection_name, indexes in active_indexes.iteritems():
+        collection = database[collection_name]
+        collection_indexes = collection.index_information().keys()
+
+        for index_name, index_data in indexes.iteritems():
+            if not index_name in collection_indexes:
+                # Get a copy actually so we don't modify the actual
+                # structure
+                index_data = copy.copy(index_data)
+                index = index_data.pop('index')
+                collection.create_index(
+                    index, name=index_name, **index_data)
+
+                indexes_added.append((collection_name, index_name))
+
+    return indexes_added
+
+
+def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES):
+    """
+    Remove any deprecated indexes from the database.
+
+    Args:
+     - database: pymongo or mongokit database instance.
+     - deprecated_indexes: the indexes to deprecate in the pattern of:
+       {'collection': ['index_identifier1', 'index_identifier2']}
+
+    Returns:
+      A list of indexes removed in form ('collection', 'index_name')
+    """
+    indexes_removed = []
+
+    for collection_name, index_names in deprecated_indexes.iteritems():
+        collection = database[collection_name]
+        collection_indexes = collection.index_information().keys()
+
+        for index_name in index_names:
+            if index_name in collection_indexes:
+                collection.drop_index(index_name)
+
+                indexes_removed.append((collection_name, index_name))
+
+    return indexes_removed

mediagoblin/gmg_commands/migrate.py

@@ -16,6 +16,7 @@
 
 
 from mediagoblin.db import migrations
+from mediagoblin.db import util as db_util
 from mediagoblin.gmg_commands import util as commands_util
 
 
@@ -27,8 +28,17 @@ def migrate_parser_setup(subparser):
 
 def migrate(args):
     mgoblin_app = commands_util.setup_app(args)
-    print "Applying migrations..."
 
+    # Clear old indexes
+    print "== Clearing old indexes... =="
+    removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db)
+
+    for collection, index_name in removed_indexes:
+        print "Removed index '%s' in collection '%s'" % (
+            index_name, collection)
+    
+    # Migrate
+    print "== Applying migrations... =="
     for model_name in migrations.MIGRATE_CLASSES:
         model = getattr(mgoblin_app.db, model_name)
 
@@ -38,4 +48,10 @@ def migrate(args):
         migration = model.migration_handler(model)
         migration.migrate_all(collection=model.collection)
             
-    print "... done."
+    # Add new indexes
+    print "== Adding new indexes... =="
+    new_indexes = db_util.add_new_indexes(mgoblin_app.db)
+
+    for collection, index_name in new_indexes:
+        print "Added index '%s' to collection '%s'" % (
+            index_name, collection)