Merge branch 'master' into derek-moore-bug405_email_notifications_for_comments

Conflicts:
	mediagoblin/db/mongo/migrations.py

mediagoblin/db/mongo/migrations.py

@@ -155,6 +155,22 @@ def convert_video_media_data(database):
         collection.save(document)
 
 @RegisterMigration(11)
+def convert_gps_media_data(database):
+    """
+    Move media_data["gps"]["*"] to media_data["gps_*"].
+    In preparation for media_data.gps_*
+    """
+    collection = database['media_entries']
+    target = collection.find(
+        {'media_data.gps': {'$exists': True}})
+
+    for document in target:
+        for key, value in document['media_data']['gps'].iteritems():
+            document['media_data']['gps_' + key] = value
+        del document['media_data']['gps']
+        collection.save(document)
+
+@RegisterMigration(12)
 def user_add_wants_comment_notification(database):
     """
     Add wants_comment_notification to user model

mediagoblin/db/mongo/util.py

@@ -310,3 +310,9 @@ def check_media_slug_used(db, uploader_id, slug, ignore_m_id):
     existing_user_slug_entries = db.MediaEntry.find(
         query_dict).count()
     return existing_user_slug_entries
+
+
+def media_entries_for_tag_slug(db, tag_slug):
+    return db.MediaEntry.find(
+        {u'state': u'processed',
+         u'tags.slug': tag_slug})

mediagoblin/db/sql/convert.py

@@ -14,12 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from copy import copy
 
 from mediagoblin.init import setup_global_and_app_config, setup_database
 from mediagoblin.db.mongo.util import ObjectId
 
 from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment,
-    Tag, MediaTag, MediaFile, MediaAttachmentFile)
+    Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData)
+from mediagoblin.media_types.image.models import ImageData
 from mediagoblin.media_types.video.models import VideoData
 from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
     sql_connect
@@ -106,6 +108,38 @@ def convert_media_entries(mk_db):
     session.close()
 
 
+def convert_image(mk_db):
+    session = Session()
+
+    for media in mk_db.MediaEntry.find(
+            {'media_type': 'mediagoblin.media_types.image'}).sort('created'):
+        media_data = copy(media.media_data)
+
+        # TODO: Fix after exif is migrated
+        media_data.pop('exif', None)
+
+        if len(media_data):
+            media_data_row = ImageData(**media_data)
+            media_data_row.media_entry = obj_id_table[media._id]
+            session.add(media_data_row)
+
+    session.commit()
+    session.close()
+
+
+def convert_video(mk_db):
+    session = Session()
+
+    for media in mk_db.MediaEntry.find(
+            {'media_type': 'mediagoblin.media_types.video'}).sort('created'):
+        media_data_row = VideoData(**media.media_data)
+        media_data_row.media_entry = obj_id_table[media._id]
+        session.add(media_data_row)
+
+    session.commit()
+    session.close()
+
+
 def convert_media_tags(mk_db):
     session = Session()
     session.autoflush = False
@@ -155,6 +189,20 @@ def convert_media_comments(mk_db):
     session.close()
 
 
+def convert_add_migration_versions():
+    session = Session()
+
+    for name in ("__main__",
+                 "mediagoblin.media_types.image",
+                 "mediagoblin.media_types.video",
+                 ):
+        m = MigrationData(name=name, version=0)
+        session.add(m)
+
+    session.commit()
+    session.close()
+
+
 def run_conversion(config_name):
     global_config, app_config = setup_global_and_app_config(config_name)
 
@@ -167,10 +215,16 @@ def run_conversion(config_name):
     Session.remove()
     convert_media_entries(mk_db)
     Session.remove()
+    convert_image(mk_db)
+    Session.remove()
+    convert_video(mk_db)
+    Session.remove()
     convert_media_tags(mk_db)
     Session.remove()
     convert_media_comments(mk_db)
     Session.remove()
+    convert_add_migration_versions()
+    Session.remove()
 
 
 if __name__ == '__main__':

mediagoblin/db/sql/models.py

@@ -20,6 +20,7 @@ TODO: indexes on foreignkeys, where useful.
 
 
 import datetime
+import sys
 
 from sqlalchemy import (
     Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey,
@@ -28,10 +29,12 @@ from sqlalchemy.orm import relationship
 from sqlalchemy.orm.collections import attribute_mapped_collection
 from sqlalchemy.sql.expression import desc
 from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.util import memoized_property
 
 from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded
 from mediagoblin.db.sql.base import Base, DictReadAttrProxy
 from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin
+from mediagoblin.db.sql.base import Session
 
 # It's actually kind of annoying how sqlalchemy-migrate does this, if
 # I understand it right, but whatever.  Anyway, don't remove this :P
@@ -58,7 +61,7 @@ class User(Base, UserMixin):
     TODO: We should consider moving some rarely used fields
     into some sort of "shadow" table.
     """
-    __tablename__ = "users"
+    __tablename__ = "core__users"
 
     id = Column(Integer, primary_key=True)
     username = Column(Unicode, nullable=False, unique=True)
@@ -85,10 +88,10 @@ class MediaEntry(Base, MediaEntryMixin):
     """
     TODO: Consider fetching the media_files using join
     """
-    __tablename__ = "media_entries"
+    __tablename__ = "core__media_entries"
 
     id = Column(Integer, primary_key=True)
-    uploader = Column(Integer, ForeignKey('users.id'), nullable=False)
+    uploader = Column(Integer, ForeignKey('core__users.id'), nullable=False)
     title = Column(Unicode, nullable=False)
     slug = Column(Unicode)
     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
@@ -168,14 +171,39 @@ class MediaEntry(Base, MediaEntryMixin):
         if media is not None:
             return media.url_for_self(urlgen)
 
+    #@memoized_property
     @property
     def media_data(self):
-        # TODO: Replace with proper code to read the correct table
-        return {}
+        session = Session()
+
+        return session.query(self.media_data_table).filter_by(
+            media_entry=self.id).first()
 
     def media_data_init(self, **kwargs):
-        # TODO: Implement this
-        pass
+        """
+        Initialize or update the contents of a media entry's media_data row
+        """
+        session = Session()
+
+        media_data = session.query(self.media_data_table).filter_by(
+            media_entry=self.id).first()
+
+        # No media data, so actually add a new one
+        if not media_data:
+            media_data = self.media_data_table(
+                **kwargs)
+            session.add(media_data)
+        # Update old media data
+        else:
+            for field, value in kwargs.iteritems():
+                setattr(media_data, field, value)
+
+    @memoized_property
+    def media_data_table(self):
+        # TODO: memoize this
+        models_module = self.media_type + '.models'
+        __import__(models_module)
+        return sys.modules[models_module].DATA_MODEL
 
 
 class FileKeynames(Base):
@@ -203,7 +231,7 @@ class MediaFile(Base):
     TODO: Highly consider moving "name" into a new table.
     TODO: Consider preloading said table in software
     """
-    __tablename__ = "mediafiles"
+    __tablename__ = "core__mediafiles"
 
     media_entry = Column(
         Integer, ForeignKey(MediaEntry.id),
@@ -242,7 +270,7 @@ class MediaAttachmentFile(Base):
 
 
 class Tag(Base):
-    __tablename__ = "tags"
+    __tablename__ = "core__tags"
 
     id = Column(Integer, primary_key=True)
     slug = Column(Unicode, nullable=False, unique=True)
@@ -259,13 +287,13 @@ class Tag(Base):
 
 
 class MediaTag(Base):
-    __tablename__ = "media_tags"
+    __tablename__ = "core__media_tags"
 
     id = Column(Integer, primary_key=True)
     media_entry = Column(
         Integer, ForeignKey(MediaEntry.id),
         nullable=False)
-    tag = Column(Integer, ForeignKey('tags.id'), nullable=False)
+    tag = Column(Integer, ForeignKey('core__tags.id'), nullable=False)
     name = Column(Unicode)
     # created = Column(DateTime, nullable=False, default=datetime.datetime.now)
 
@@ -292,12 +320,12 @@ class MediaTag(Base):
 
 
 class MediaComment(Base, MediaCommentMixin):
-    __tablename__ = "media_comments"
+    __tablename__ = "core__media_comments"
 
     id = Column(Integer, primary_key=True)
     media_entry = Column(
-        Integer, ForeignKey('media_entries.id'), nullable=False)
-    author = Column(Integer, ForeignKey('users.id'), nullable=False)
+        Integer, ForeignKey('core__media_entries.id'), nullable=False)
+    author = Column(Integer, ForeignKey('core__users.id'), nullable=False)
     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
     content = Column(UnicodeText, nullable=False)
 
@@ -319,7 +347,7 @@ MODELS = [
 ######################################################
 
 class MigrationData(Base):
-    __tablename__ = "migrations"
+    __tablename__ = "core__migrations"
 
     name = Column(Unicode, primary_key=True)
     version = Column(Integer, nullable=False, default=0)

mediagoblin/db/sql/util.py

@@ -294,6 +294,15 @@ def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
     return does_exist
 
 
+def media_entries_for_tag_slug(dummy_db, tag_slug):
+    return MediaEntry.query \
+        .join(MediaEntry.tags_helper) \
+        .join(MediaTag.tag_helper) \
+        .filter(
+            (MediaEntry.state == u'processed')
+            & (Tag.slug == tag_slug))
+
+
 def clean_orphan_tags():
     q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
     for t in q1:

mediagoblin/db/util.py

@@ -21,7 +21,9 @@ except ImportError:
 
 if use_sql:
     from mediagoblin.db.sql.fake import ObjectId, InvalidId, DESCENDING
-    from mediagoblin.db.sql.util import atomic_update, check_media_slug_used
+    from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \
+        media_entries_for_tag_slug
 else:
     from mediagoblin.db.mongo.util import \
-        ObjectId, InvalidId, DESCENDING, atomic_update, check_media_slug_used
+        ObjectId, InvalidId, DESCENDING, atomic_update, \
+        check_media_slug_used, media_entries_for_tag_slug