Add Graveyard model

This adds the Graveyard model which is used when a model is deleted, it
stores the important "shell" information on the model so it can
hard-delete the real object. It also remaps the GenericModelReference
references to the new Graveyard model.

This also moves the soft deletion setting from __model_args__ to
"deletion_mode" on the model.

mediagoblin/db/base.py

@@ -13,8 +13,6 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import datetime
-
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import inspect
 
@@ -30,11 +28,7 @@ class GMGTableBase(object):
     HARD_DELETE = "hard-deletion"
     SOFT_DELETE = "soft-deletion"
 
-    __default_model_args__ = {
+    deletion_mode = HARD_DELETE
-        "deletion": HARD_DELETE,
-        "soft_deletion_field": "deleted",
-        "soft_deletion_retain": ("id",)
-    }
 
     @property
     def _session(self):
@@ -47,11 +41,6 @@ class GMGTableBase(object):
     if not DISABLE_GLOBALS:
         query = Session.query_property()
 
-    def get_model_arg(self, argument):
-        model_args = self.__default_model_args__.copy()
-        model_args.update(getattr(self, "__model_args__", {}))
-        return model_args.get(argument)
-
     def get(self, key):
         return getattr(self, key)
 
@@ -70,42 +59,54 @@ class GMGTableBase(object):
         else:
             sess.flush()
 
-    def delete(self, commit=True):
+    def delete(self, commit=True, deletion=None):
         """ Delete the object either using soft or hard deletion """
-        if self.get_model_arg("deletion") == self.HARD_DELETE:
+        # Get the setting in the model args if none has been specified.
-            return self.hard_delete(commit)
+        if deletion is None:
-        elif self.get_model_arg("deletion") == self.SOFT_DELETE:
+            deletion = self.deletion_mode
-            return self.soft_delete(commit)
+
+        # Hand off to the correct deletion function.
+        if deletion == self.HARD_DELETE:
+            return self.hard_delete(commit=commit)
+        elif deletion == self.SOFT_DELETE:
+            return self.soft_delete(commit=commit)
         else:
             raise ValueError(
-                "__model_args__['deletion'] is an invalid value %s" % (
+                "Invalid deletion mode {mode!r}".format(
-                    self.get_model_arg("deletion")
+                    mode=deletion
-                ))
+                )
+            )
 
     def soft_delete(self, commit):
-        # Find the deletion field
+        # Create the graveyard version of this model
-        field_name = self.get_model_arg("soft_deletion_field")
+        # Importing this here due to cyclic imports
+        from mediagoblin.db.models import User, Graveyard, GenericModelReference
+        tombstone = Graveyard()
+        if getattr(self, "public_id", None) is not None:
+            tombstone.public_id = self.public_id
 
-        # We can't use self.__table__.columns as it only shows it of the
+        # This is a special case, we don't want to save any actor if the thing
-        # current model and no parent if polymorphism is being used. This
+        # being soft deleted is a User model as this would create circular
-        # will cause problems for example for the User model.
+        # ForeignKeys
-        if field_name not in dir(type(self)):
+        if not isinstance(self, User):
-            raise ValueError("Cannot find soft_deletion_field")
+            tombstone.actor = User.query.filter_by(
+                id=self.actor
+            ).first()
+        tombstone.object_type = self.object_type
+        tombstone.save()
 
-        # Store a value in the deletion field
+        # There will be a lot of places where the GenericForeignKey will point
-        setattr(self, field_name, datetime.datetime.utcnow())
+        # to the model, we want to remap those to our tombstone.
+        gmrs = GenericModelReference.query.filter_by(
+            obj_pk=self.id,
+            model_type=self.__tablename__
+        ).update({
+            "obj_pk": tombstone.id,
+            "model_type": tombstone.__tablename__,
+        })
 
-        # Iterate through the fields and remove data
+        # Now we can go ahead and actually delete the model.
-        retain_fields = self.get_model_arg("soft_deletion_retain")
+        return self.hard_delete(commit=commit)
-        for field_name in self.__table__.columns.keys():
-            # should we skip this field?
-            if field_name in retain_fields:
-                continue
-
-            setattr(self, field_name, None)
-
-        # Save the changes
-        self.save(commit)
 
     def hard_delete(self, commit):
         """Delete the object and commit the change immediately by default"""

mediagoblin/db/migrations.py

@@ -1821,69 +1821,28 @@ def federation_actor(db):
     # commit changes to db.
     db.commit()
 
+class Graveyard_V0(declarative_base()):
+    """ Where models come to die """
+    __tablename__ = "core__graveyard"
+
+    id = Column(Integer, primary_key=True)
+    public_id = Column(Unicode, nullable=True, unique=True)
+
+    deleted = Column(DateTime, nullable=False)
+    object_type = Column(Unicode, nullable=False)
+
+    actor_id = Column(Integer, ForeignKey(GenericModelReference_V0.id))
+
 @RegisterMigration(39, MIGRATIONS)
-def federation_soft_deletion(db):
+def federation_graveyard(db):
     """ Introduces soft deletion to models
 
-    This adds a deleted DateTime column which represents if the model is
+    This adds a Graveyard model which is used to copy (soft-)deleted models to.
-    deleted and if so, when. With this change comes changes on the models
-    that soft delete the models rather than the previous hard deletion.
     """
     metadata = MetaData(bind=db.bind)
 
-    # User Model
+    # Create the graveyard table
-    user_table = inspect_table(metadata, "core__users")
+    Graveyard_V0.__table__.create(db.bind)
-    user_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    user_deleted_column.create(user_table)
-
-    # MediaEntry
-    media_entry_table = inspect_table(metadata, "core__media_entries")
-    me_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    me_deleted_column.create(media_entry_table)
-
-    # MediaComment
-    media_comment_table = inspect_table(metadata, "core__media_comments")
-    mc_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    mc_deleted_column.create(media_comment_table)
-
-    # Collection
-    collection_table = inspect_table(metadata, "core__collections")
-    collection_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    collection_deleted_column.create(collection_table)
-
-    # Generator
-    generator_table = inspect_table(metadata, "core__generators")
-    generator_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    generator_deleted_column.create(generator_table)
-
-    # Activity
-    activity_table = inspect_table(metadata, "core__activities")
-    activity_deleted_column = Column(
-        "deleted",
-        DateTime,
-        nullable=True
-    )
-    activity_deleted_column.create(activity_table)
 
     # Commit changes to the db
     db.commit()

mediagoblin/db/models.py

@@ -243,7 +243,6 @@ class User(Base, UserMixin):
 
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
 
     location = Column(Integer, ForeignKey("core__locations.id"))
 
@@ -255,11 +254,25 @@ class User(Base, UserMixin):
         'polymorphic_on': type,
     }
 
-    __model_args__ = {
+    deletion_mode = Base.SOFT_DELETE
-        'deletion': Base.SOFT_DELETE,
-    }
 
-    def delete(self, **kwargs):
+    def soft_delete(self, *args, **kwargs):
+        # Find all the Collections and delete those
+        for collection in Collection.query.filter_by(actor=self.id):
+            collection.delete(**kwargs)
+
+        # Find all the comments and delete those too
+        for comment in MediaComment.query.filter_by(actor=self.id):
+            comment.delete(**kwargs)
+
+        # Find all the activities and delete those too
+        for activity in Activity.query.filter_by(actor=self.id):
+            activity.delete(**kwargs)
+
+        super(User, self).soft_delete(*args, **kwargs)
+
+
+    def delete(self, *args, **kwargs):
         """Deletes a User and all related entries/comments/files/..."""
         # Collections get deleted by relationships.
 
@@ -276,7 +289,7 @@ class User(Base, UserMixin):
 
         # Delete user, pass through commit=False/True in kwargs
         username = self.username
-        super(User, self).delete(**kwargs)
+        super(User, self).delete(*args, **kwargs)
         _log.info('Deleted user "{0}" account'.format(username))
 
     def has_privilege(self, privilege, allow_admin=True):
@@ -521,7 +534,6 @@ class MediaEntry(Base, MediaEntryMixin):
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow,
         index=True)
     updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
 
     fail_error = Column(Unicode)
     fail_metadata = Column(JSONEncoded)
@@ -536,6 +548,8 @@ class MediaEntry(Base, MediaEntryMixin):
         UniqueConstraint('actor', 'slug'),
         {})
 
+    deletion_mode = Base.SOFT_DELETE
+
     get_actor = relationship(User)
 
     media_files_helper = relationship("MediaFile",
@@ -673,6 +687,13 @@ class MediaEntry(Base, MediaEntryMixin):
                 id=self.id,
                 title=safe_title)
 
+    def soft_delete(self, *args, **kwargs):
+        # Find all of the media comments for this and delete them
+        for comment in MediaComment.query.filter_by(media_entry=self.id):
+            comment.delete(*args, **kwargs)
+
+        super(MediaEntry, self).soft_delete(*args, **kwargs)
+
     def delete(self, del_orphan_tags=True, **kwargs):
         """Delete MediaEntry and all related files/attachments/comments
 
@@ -915,7 +936,6 @@ class MediaComment(Base, MediaCommentMixin):
         Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
     actor = Column(Integer, ForeignKey(User.id), nullable=False)
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
     content = Column(UnicodeText, nullable=False)
     location = Column(Integer, ForeignKey("core__locations.id"))
     get_location = relationship("Location", lazy="joined")
@@ -941,9 +961,7 @@ class MediaComment(Base, MediaCommentMixin):
                                                    lazy="dynamic",
                                                    cascade="all, delete-orphan"))
 
-    __model_args__ = {
+    deletion_mode = Base.SOFT_DELETE
-        "deletion": Base.SOFT_DELETE,
-    }
 
     def serialize(self, request):
         """ Unserialize to python dictionary for API """
@@ -1021,7 +1039,6 @@ class Collection(Base, CollectionMixin):
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow,
                      index=True)
     updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
     description = Column(UnicodeText)
     actor = Column(Integer, ForeignKey(User.id), nullable=False)
     num_items = Column(Integer, default=0)
@@ -1043,9 +1060,7 @@ class Collection(Base, CollectionMixin):
         UniqueConstraint("actor", "slug"),
         {})
 
-    __model_args__ = {
+    deletion_mode = Base.SOFT_DELETE
-        "delete": Base.SOFT_DELETE,
-    }
 
     # These are the types, It's strongly suggested if new ones are invented they
     # are prefixed to ensure they're unique from other types. Any types used in
@@ -1438,12 +1453,9 @@ class Generator(Base):
     name = Column(Unicode, nullable=False)
     published = Column(DateTime, default=datetime.datetime.utcnow)
     updated = Column(DateTime, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
     object_type = Column(Unicode, nullable=False)
 
-    __model_args__ = {
+    deletion_mode = Base.SOFT_DELETE
-        "deletion": Base.SOFT_DELETE,
-    }
 
     def __repr__(self):
         return "<{klass} {name}>".format(
@@ -1485,7 +1497,6 @@ class Activity(Base, ActivityMixin):
                    nullable=False)
     published = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
-    deleted = Column(DateTime, nullable=True)
 
     verb = Column(Unicode, nullable=False)
     content = Column(Unicode, nullable=True)
@@ -1511,9 +1522,7 @@ class Activity(Base, ActivityMixin):
                                              cascade="all, delete-orphan"))
     get_generator = relationship(Generator)
 
-    __model_args__ = {
+    deletion_mode = Base.SOFT_DELETE
-        "deletion": Base.SOFT_DELETE,
-    }
 
     def __repr__(self):
         if self.content is None:
@@ -1532,6 +1541,39 @@ class Activity(Base, ActivityMixin):
             self.updated = datetime.datetime.now()
         super(Activity, self).save(*args, **kwargs)
 
+class Graveyard(Base):
+    """ Where models come to die """
+    __tablename__ = "core__graveyard"
+
+    id = Column(Integer, primary_key=True)
+    public_id = Column(Unicode, nullable=True, unique=True)
+
+    deleted = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
+    object_type = Column(Unicode, nullable=False)
+
+    # This could either be a deleted actor or a real actor, this must be
+    # nullable as it we shouldn't have it set for deleted actor
+    actor_id = Column(Integer, ForeignKey(GenericModelReference.id))
+    actor_helper = relationship(GenericModelReference)
+    actor = association_proxy("actor_helper", "get_object",
+                              creator=GenericModelReference.find_or_new)
+
+    def __repr__(self):
+        return "<{klass} deleted {obj_type}>".format(
+            klass=type(self).__name__,
+            obj_type=self.object_type
+        )
+
+    def serialize(self, request):
+        return {
+            "id": self.public_id,
+            "objectType": self.object_type,
+            "actor": self.actor(),
+            "published": self.deleted,
+            "updated": self.deleted,
+            "deleted": self.deleted
+        }
+
 with_polymorphic(
     Notification,
     [ProcessingNotification, CommentNotification])
@@ -1543,7 +1585,7 @@ MODELS = [
     ProcessingNotification, Client, CommentSubscription, ReportBase,
     CommentReport, MediaReport, UserBan, Privilege, PrivilegeUserAssociation,
     RequestToken, AccessToken, NonceTimestamp, Activity, Generator, Location,
-    GenericModelReference]
+    GenericModelReference, Graveyard]
 
 """
  Foundations are the default rows that are created immediately after the tables