Merge branch 'master' into sqlmigrate

Conflicts:
	mediagoblin/db/sql/models.py

mediagoblin/db/__init__.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/mixin.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,6 +29,7 @@ real objects.
 
 from mediagoblin.auth import lib as auth_lib
 from mediagoblin.tools import common, licenses
+from mediagoblin.tools.text import cleaned_markdown_conversion
 
 
 class UserMixin(object):
@@ -39,8 +40,20 @@ class UserMixin(object):
         return auth_lib.bcrypt_check_password(
             password, self.pw_hash)
 
+    @property
+    def bio_html(self):
+        return cleaned_markdown_conversion(self.bio)
+
 
 class MediaEntryMixin(object):
+    @property
+    def description_html(self):
+        """
+        Rendered version of the description, run through
+        Markdown and cleaned with our cleaning tool.
+        """
+        return cleaned_markdown_conversion(self.description)
+
     def get_display_media(self, media_map,
                           fetch_order=common.DISPLAY_IMAGE_FETCHING_ORDER):
         """
@@ -91,3 +104,13 @@ class MediaEntryMixin(object):
     def get_license_data(self):
         """Return license dict for requested license"""
         return licenses.SUPPORTED_LICENSES[self.license or ""]
+
+
+class MediaCommentMixin(object):
+    @property
+    def content_html(self):
+        """
+        the actual html-rendered version of the comment displayed.
+        Run through Markdown and the HTML cleaner.
+        """
+        return cleaned_markdown_conversion(self.content)

mediagoblin/db/mongo/__init__.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/mongo/indexes.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/mongo/migrations.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,6 +29,16 @@ def add_table_field(db, table_name, field_name, default_value):
         multi=True)
 
 
+def drop_table_field(db, table_name, field_name):
+    """
+    Drop an old field from a table/collection
+    """
+    db[table_name].update(
+        {field_name: {'$exists': True}},
+        {'$unset': {field_name: 1}},
+        multi=True)
+
+
 # Please see mediagoblin/tests/test_migrations.py for some examples of
 # basic migrations.
 
@@ -115,3 +125,17 @@ def mediaentry_add_license(database):
     Add the 'license' field for entries that don't have it.
     """
     add_table_field(database, 'media_entries', 'license', None)
+
+
+@RegisterMigration(9)
+def remove_calculated_html(database):
+    """
+    Drop pre-rendered html again and calculate things
+    on the fly (and cache):
+    - User.bio_html
+    - MediaEntry.description_html
+    - MediaComment.content_html
+    """
+    drop_table_field(database, 'users', 'bio_html')
+    drop_table_field(database, 'media_entries', 'description_html')
+    drop_table_field(database, 'media_comments', 'content_html')

mediagoblin/db/mongo/models.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,7 +23,7 @@ from mediagoblin.db.mongo import migrations
 from mediagoblin.db.mongo.util import ASCENDING, DESCENDING, ObjectId
 from mediagoblin.tools.pagination import Pagination
 from mediagoblin.tools import url
-from mediagoblin.db.mixin import UserMixin, MediaEntryMixin
+from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin
 
 ###################
 # Custom validators
@@ -59,7 +59,6 @@ class User(Document, UserMixin):
      - is_admin: Whether or not this user is an administrator or not.
      - url: this user's personal webpage/website, if appropriate.
      - bio: biography of this user (plaintext, in markdown)
-     - bio_html: biography of the user converted to proper HTML.
     """
     __collection__ = 'users'
     use_dot_notation = True
@@ -76,7 +75,6 @@ class User(Document, UserMixin):
         'is_admin': bool,
         'url': unicode,
         'bio': unicode,      # May contain markdown
-        'bio_html': unicode,  # May contain plaintext, or HTML
         'fp_verification_key': unicode,  # forgotten password verification key
         'fp_token_expire': datetime.datetime,
         }
@@ -112,9 +110,6 @@ class MediaEntry(Document, MediaEntryMixin):
        up with MarkDown for slight fanciness (links, boldness, italics,
        paragraphs...)
 
-     - description_html: Rendered version of the description, run through
-       Markdown and cleaned with our cleaning tool.
-
      - media_type: What type of media is this?  Currently we only support
        'image' ;)
 
@@ -179,7 +174,6 @@ class MediaEntry(Document, MediaEntryMixin):
         'slug': unicode,
         'created': datetime.datetime,
         'description': unicode,  # May contain markdown/up
-        'description_html': unicode,  # May contain plaintext, or HTML
         'media_type': unicode,
         'media_data': dict,  # extra data relevant to this media_type
         'plugin_data': dict,  # plugins can dump stuff here.
@@ -257,7 +251,7 @@ class MediaEntry(Document, MediaEntryMixin):
         return self.db.User.find_one({'_id': self.uploader})
 
 
-class MediaComment(Document):
+class MediaComment(Document, MediaCommentMixin):
     """
     A comment on a MediaEntry.
 
@@ -266,8 +260,6 @@ class MediaComment(Document):
      - author: user who posted this comment
      - created: when the comment was created
      - content: plaintext (but markdown'able) version of the comment's content.
-     - content_html: the actual html-rendered version of the comment displayed.
-       Run through Markdown and the HTML cleaner.
     """
 
     __collection__ = 'media_comments'
@@ -278,7 +270,7 @@ class MediaComment(Document):
         'author': ObjectId,
         'created': datetime.datetime,
         'content': unicode,
-        'content_html': unicode}
+        }
 
     required_fields = [
         'media_entry', 'author', 'created', 'content']

mediagoblin/db/mongo/open.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/mongo/util.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/open.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/sql/__init__.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/sql/base.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/sql/convert.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -56,7 +56,7 @@ def convert_users(mk_db):
         copy_attrs(entry, new_entry,
             ('username', 'email', 'created', 'pw_hash', 'email_verified',
              'status', 'verification_key', 'is_admin', 'url',
-             'bio', 'bio_html',
+             'bio',
              'fp_verification_key', 'fp_token_expire',))
         # new_entry.fp_verification_expire = entry.fp_token_expire
 
@@ -77,7 +77,7 @@ def convert_media_entries(mk_db):
         new_entry = MediaEntry()
         copy_attrs(entry, new_entry,
             ('title', 'slug', 'created',
-             'description', 'description_html',
+             'description',
              'media_type', 'state', 'license',
              'fail_error',
              'queued_task_id',))
@@ -133,7 +133,7 @@ def convert_media_comments(mk_db):
         new_entry = MediaComment()
         copy_attrs(entry, new_entry,
             ('created',
-             'content', 'content_html',))
+             'content',))
         copy_reference_attr(entry, new_entry, "media_entry")
         copy_reference_attr(entry, new_entry, "author")
 

mediagoblin/db/sql/extratypes.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -25,8 +25,10 @@ class PathTupleWithSlashes(TypeDecorator):
 
     def process_bind_param(self, value, dialect):
         if value is not None:
-            assert len(value), "Does not support empty lists"
-            value = '/'.join(value)
+            if len(value) == 0:
+                value = None
+            else:
+                value = '/'.join(value)
         return value
 
     def process_result_value(self, value, dialect):

mediagoblin/db/sql/fake.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/sql/models.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,6 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+"""
+TODO: indexes on foreignkeys, where useful.
+"""
+
 
 import datetime
 
@@ -27,7 +31,7 @@ from sqlalchemy.ext.associationproxy import association_proxy
 
 from mediagoblin.db.sql.extratypes import PathTupleWithSlashes
 from mediagoblin.db.sql.base import Base, DictReadAttrProxy
-from mediagoblin.db.mixin import UserMixin, MediaEntryMixin
+from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin
 
 # It's actually kind of annoying how sqlalchemy-migrate does this, if
 # I understand it right, but whatever.  Anyway, don't remove this :P
@@ -50,6 +54,10 @@ class SimpleFieldAlias(object):
 
 
 class User(Base, UserMixin):
+    """
+    TODO: We should consider moving some rarely used fields
+    into some sort of "shadow" table.
+    """
     __tablename__ = "users"
 
     id = Column(Integer, primary_key=True)
@@ -57,13 +65,12 @@ class User(Base, UserMixin):
     email = Column(Unicode, nullable=False)
     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
     pw_hash = Column(Unicode, nullable=False)
-    email_verified = Column(Boolean)
+    email_verified = Column(Boolean, default=False)
     status = Column(Unicode, default=u"needs_email_verification", nullable=False)
     verification_key = Column(Unicode)
     is_admin = Column(Boolean, default=False, nullable=False)
     url = Column(Unicode)
     bio = Column(UnicodeText)  # ??
-    bio_html = Column(UnicodeText)  # ??
     fp_verification_key = Column(Unicode)
     fp_token_expire = Column(DateTime)
 
@@ -74,6 +81,9 @@ class User(Base, UserMixin):
 
 
 class MediaEntry(Base, MediaEntryMixin):
+    """
+    TODO: Consider fetching the media_files using join
+    """
     __tablename__ = "media_entries"
 
     id = Column(Integer, primary_key=True)
@@ -82,9 +92,9 @@ class MediaEntry(Base, MediaEntryMixin):
     slug = Column(Unicode)
     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
     description = Column(UnicodeText) # ??
-    description_html = Column(UnicodeText) # ??
     media_type = Column(Unicode, nullable=False)
-    state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum?
+    state = Column(Unicode, default=u'unprocessed', nullable=False)
+        # or use sqlalchemy.types.Enum?
     license = Column(Unicode)
 
     fail_error = Column(Unicode)
@@ -120,6 +130,8 @@ class MediaEntry(Base, MediaEntryMixin):
     # attachment_files
     # fail_error
 
+    _id = SimpleFieldAlias("id")
+
     def get_comments(self, ascending=False):
         order_col = MediaComment.created
         if not ascending:
@@ -149,6 +161,10 @@ class MediaEntry(Base, MediaEntryMixin):
 
 
 class MediaFile(Base):
+    """
+    TODO: Highly consider moving "name" into a new table.
+    TODO: Consider preloading said table in software
+    """
     __tablename__ = "mediafiles"
 
     media_entry = Column(
@@ -209,7 +225,7 @@ class MediaTag(Base):
         return DictReadAttrProxy(self)
 
 
-class MediaComment(Base):
+class MediaComment(Base, MediaCommentMixin):
     __tablename__ = "media_comments"
 
     id = Column(Integer, primary_key=True)
@@ -218,10 +234,11 @@ class MediaComment(Base):
     author = Column(Integer, ForeignKey('users.id'), nullable=False)
     created = Column(DateTime, nullable=False, default=datetime.datetime.now)
     content = Column(UnicodeText, nullable=False)
-    content_html = Column(UnicodeText)
 
     get_author = relationship(User)
 
+    _id = SimpleFieldAlias("id")
+
 
 MODELS = [
     User, MediaEntry, Tag, MediaTag, MediaComment]
@@ -243,12 +260,20 @@ class MigrationData(Base):
 ######################################################
 
 
-def show_table_init():
+def show_table_init(engine_uri):
+    if engine_uri is None:
+        engine_uri = 'sqlite:///:memory:'
     from sqlalchemy import create_engine
-    engine = create_engine('sqlite:///:memory:', echo=True)
+    engine = create_engine(engine_uri, echo=True)
 
     Base.metadata.create_all(engine)
 
 
 if __name__ == '__main__':
-    show_table_init()
+    from sys import argv
+    print repr(argv)
+    if len(argv) == 2:
+        uri = argv[1]
+    else:
+        uri = None
+    show_table_init(uri)

mediagoblin/db/sql/open.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011,2012 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mediagoblin/db/util.py

@@ -1,5 +1,5 @@
 # GNU MediaGoblin -- federated, autonomous media hosting
-# Copyright (C) 2011 MediaGoblin contributors.  See AUTHORS.
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by