Merge remote-tracking branch 'refs/remotes/rodney757/file_limits'

Conflicts:
	mediagoblin/db/migrations.py

mediagoblin/config_spec.ini

@@ -75,6 +75,12 @@ theme = string()
 plugin_web_path = string(default="/plugin_static/")
 plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/")
 
+# Default user upload limit (in Mb)
+upload_limit = integer(default=None)
+
+# Max file size (in Mb)
+max_file_size = integer(default=None)
+
 [jinja2]
 # Jinja2 supports more directives than the minimum required by mediagoblin. 
 # This setting allows users creating custom templates to specify a list of

mediagoblin/db/migrations.py

@@ -474,3 +474,23 @@ def wants_notifications(db):
     col.create(user_table)
 
     db.commit()
+
+
+@RegisterMigration(16, MIGRATIONS)
+def upload_limits(db):
+    """Add user upload limit columns"""
+    metadata = MetaData(bind=db.bind)
+
+    user_table = inspect_table(metadata, 'core__users')
+    media_entry_table = inspect_table(metadata, 'core__media_entries')
+
+    col = Column('uploaded', Integer, default=0)
+    col.create(user_table)
+
+    col = Column('upload_limit', Integer)
+    col.create(user_table)
+
+    col = Column('file_size', Integer, default=0)
+    col.create(media_entry_table)
+
+    db.commit()

mediagoblin/db/models.py

@@ -74,6 +74,8 @@ class User(Base, UserMixin):
     is_admin = Column(Boolean, default=False, nullable=False)
     url = Column(Unicode)
     bio = Column(UnicodeText)  # ??
+    uploaded = Column(Integer, default=0)
+    upload_limit = Column(Integer)
 
     ## TODO
     # plugin data would be in a separate model
@@ -190,6 +192,7 @@ class MediaEntry(Base, MediaEntryMixin):
         # or use sqlalchemy.types.Enum?
     license = Column(Unicode)
     collected = Column(Integer, default=0)
+    file_size = Column(Integer, default=0)
 
     fail_error = Column(Unicode)
     fail_metadata = Column(JSONEncoded)

mediagoblin/static/js/file_size.js

@@ -0,0 +1,45 @@
+/**
+ * GNU MediaGoblin -- federated, autonomous media hosting
+ * Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+$(document).ready(function(){
+    var file = document.getElementById('file');
+    var uploaded = parseInt(document.getElementById('uploaded').value);
+    var upload_limit = parseInt(document.getElementById('upload_limit').value);
+    var max_file_size = parseInt(document.getElementById('max_file_size').value);
+
+    file.onchange = function() {
+        var file_size = file.files[0].size / (1024.0 * 1024);
+
+        if (file_size >= max_file_size) {
+            $('#file').after('<p id="file_size_error" class="form_field_error">Sorry, the file size is too big.</p>'); 
+        }
+        else if (document.getElementById('file_size_error')) {
+            $('#file_size_error').hide();
+        }
+
+        if (upload_limit) {
+            if ( uploaded + file_size >= upload_limit) {
+                $('#file').after('<p id="upload_limit_error" class="form_field_error">Sorry, uploading this file will put you over your upload limit.</p>');
+            }
+            else if (document.getElementById('upload_limit_error')) {
+                $('#upload_limit_error').hide();
+            console.log(file_size >= max_file_size);
+            }
+        }
+    };
+});

mediagoblin/storage/__init__.py

@@ -191,6 +191,13 @@ class StorageInterface(object):
                 # Copy to storage system in 4M chunks
                 shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 
+    def get_file_size(self, filepath):
+        """
+        Return the size of the file in bytes.
+        """
+        # Subclasses should override this method.
+        self.__raise_not_implemented()
+
 
 ###########
 # Utilities

mediagoblin/storage/cloudfiles.py

@@ -168,6 +168,12 @@ class CloudFilesStorage(StorageInterface):
                 # Copy to storage system in 4096 byte chunks
                 dest_file.send(source_file)
 
+    def get_file_size(self, filepath):
+        """Returns the file size in bytes"""
+        obj = self.container.get_object(
+                self._resolve_filepath(filepath))
+        return obj.total_bytes
+
 class CloudFilesStorageObjectWrapper():
     """
     Wrapper for python-cloudfiles's cloudfiles.storage_object.Object

mediagoblin/storage/filestorage.py

@@ -111,3 +111,6 @@ class BasicFileStorage(StorageInterface):
                 os.makedirs(directory)
         # This uses chunked copying of 16kb buffers (Py2.7):
         shutil.copy(filename, self.get_local_path(filepath))
+
+    def get_file_size(self, filepath):
+        return os.stat(self._resolve_filepath(filepath)).st_size

mediagoblin/submit/forms.py

@@ -17,13 +17,22 @@
 
 import wtforms
 
+from mediagoblin import mg_globals
 from mediagoblin.tools.text import tag_length_validator
 from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
 from mediagoblin.tools.licenses import licenses_as_choices
 
 
-class SubmitStartForm(wtforms.Form):
+def get_submit_start_form(form, **kwargs):
-    file = wtforms.FileField(_('File'))
+    max_file_size = kwargs.get('max_file_size')
+    desc = None
+    if max_file_size:
+        desc = _('Max file size: {0} mb'.format(max_file_size))
+
+    class SubmitStartForm(wtforms.Form):
+        file = wtforms.FileField(
+            _('File'),
+            description=desc)
         title = wtforms.TextField(
             _('Title'),
             [wtforms.validators.Length(min=0, max=500)])
@@ -41,6 +50,11 @@ class SubmitStartForm(wtforms.Form):
             _('License'),
             [wtforms.validators.Optional(),],
             choices=licenses_as_choices())
+        max_file_size = wtforms.HiddenField('')
+        upload_limit = wtforms.HiddenField('')
+        uploaded = wtforms.HiddenField('')
+
+    return SubmitStartForm(form, **kwargs)
 
 class AddCollectionForm(wtforms.Form):
     title = wtforms.TextField(

mediagoblin/submit/views.py

@@ -43,8 +43,28 @@ def submit_start(request):
     """
     First view for submitting a file.
     """
-    submit_form = submit_forms.SubmitStartForm(request.form,
+    user = request.user
-        license=request.user.license_preference)
+    if user.upload_limit >= 0:
+        upload_limit = user.upload_limit
+    else:
+        upload_limit = mg_globals.app_config.get('upload_limit', None)
+
+    if upload_limit and user.uploaded >= upload_limit:
+        messages.add_message(
+            request,
+            messages.WARNING,
+            _('Sorry, you have reached your upload limit.'))
+        return redirect(request, "mediagoblin.user_pages.user_home",
+                        user=request.user.username)
+
+    max_file_size = mg_globals.app_config.get('max_file_size', None)
+
+    submit_form = submit_forms.get_submit_start_form(
+        request.form,
+        license=request.user.license_preference,
+        max_file_size=max_file_size,
+        upload_limit=upload_limit,
+        uploaded=user.uploaded)
 
     if request.method == 'POST' and submit_form.validate():
         if not check_file_field(request, 'file'):
@@ -86,10 +106,36 @@ def submit_start(request):
                 with queue_file:
                     queue_file.write(request.files['file'].stream.read())
 
+                # Get file size and round to 2 decimal places
+                file_size = request.app.queue_store.get_file_size(
+                    entry.queued_media_file) / (1024.0 * 1024)
+                file_size = float('{0:.2f}'.format(file_size))
+
+                error = False
+
+                # Check if file size is over the limit
+                if max_file_size and file_size >= max_file_size:
+                    submit_form.file.errors.append(
+                        _(u'Sorry, the file size is too big.'))
+                    error = True
+
+                # Check if user is over upload limit
+                if upload_limit and (user.uploaded + file_size) >= upload_limit:
+                    submit_form.file.errors.append(
+                        _('Sorry, uploading this file will put you over your'
+                          ' upload limit.'))
+                    error = True
+
+                if not error:
+                    user.uploaded = user.uploaded + file_size
+                    user.save()
+
+                    entry.file_size = file_size
+
                     # Save now so we have this data before kicking off processing
                     entry.save()
 
-                # Pass off to async processing
+                    # Pass off to processing
                     #
                     # (... don't change entry after this point to avoid race
                     # conditions with changes to the document via processing code)
@@ -97,13 +143,12 @@ def submit_start(request):
                         'mediagoblin.user_pages.atom_feed',
                         qualified=True, user=request.user.username)
                     run_process_media(entry, feed_url)
-
                     add_message(request, SUCCESS, _('Woohoo! Submitted!'))
 
                     add_comment_subscription(request.user, entry)
 
                     return redirect(request, "mediagoblin.user_pages.user_home",
-                                user=request.user.username)
+                                user=user.username)
             except Exception as e:
                 '''
                 This section is intended to catch exceptions raised in

mediagoblin/templates/mediagoblin/submit/start.html

@@ -19,6 +19,11 @@
 
 {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %}
 
+{% block mediagoblin_head %}
+  <script type="text/javascript"
+          src="{{ request.staticdirect('/js/file_size.js') }}"></script>
+{% endblock %}
+
 {% block title -%}
   {% trans %}Add your media{% endtrans %} &mdash; {{ super() }}
 {%- endblock %}

mediagoblin/tests/resources.py

@@ -29,6 +29,8 @@ EVIL_JPG = resource('evil.jpg')
 EVIL_PNG = resource('evil.png')
 BIG_BLUE = resource('bigblue.png')
 GOOD_PDF = resource('good.pdf')
+MED_PNG = resource('medium.png')
+BIG_PNG = resource('big.png')
 
 
 def resource_exif(f):

mediagoblin/tests/test_mgoblin_app.ini

@@ -13,6 +13,10 @@ tags_max_length = 50
 # So we can start to test attachments:
 allow_attachments = True
 
+upload_limit = 500
+
+max_file_size = 2
+
 [storage:publicstore]
 base_dir = %(here)s/user_dev/media/public
 base_url = /mgoblin_media/

mediagoblin/tests/test_submission.py

@@ -24,13 +24,14 @@ import pytest
 
 from mediagoblin.tests.tools import fixture_add_user
 from mediagoblin import mg_globals
-from mediagoblin.db.models import MediaEntry
+from mediagoblin.db.models import MediaEntry, User
+from mediagoblin.db.base import Session
 from mediagoblin.tools import template
 from mediagoblin.media_types.image import ImageMediaManager
 from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites
 
 from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \
-    BIG_BLUE, GOOD_PDF, GPS_JPG
+    BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG
 
 GOOD_TAG_STRING = u'yin,yang'
 BAD_TAG_STRING = unicode('rage,' + 'f' * 26 + 'u' * 26)
@@ -107,9 +108,38 @@ class TestSubmission:
         self.logout()
         self.test_app.get(url)
 
+    def user_upload_limits(self, uploaded=None, upload_limit=None):
+        if uploaded:
+            self.test_user.uploaded = uploaded
+        if upload_limit:
+            self.test_user.upload_limit = upload_limit
+
+        self.test_user.save()
+
+        # Reload
+        self.test_user = User.query.filter_by(
+            username=self.test_user.username
+        ).first()
+
+        # ... and detach from session:
+        Session.expunge(self.test_user)
+
     def test_normal_jpg(self):
+        # User uploaded should be 0
+        assert self.test_user.uploaded == 0
+
         self.check_normal_upload(u'Normal upload 1', GOOD_JPG)
 
+        # User uploaded should be the same as GOOD_JPG size in Mb
+        file_size = os.stat(GOOD_JPG).st_size / (1024.0 * 1024)
+        file_size = float('{0:.2f}'.format(file_size))
+
+        # Reload user
+        self.test_user = User.query.filter_by(
+            username=self.test_user.username
+        ).first()
+        assert self.test_user.uploaded == file_size
+
     def test_normal_png(self):
         self.check_normal_upload(u'Normal upload 2', GOOD_PNG)
 
@@ -121,6 +151,75 @@ class TestSubmission:
         self.check_url(response, '/u/{0}/'.format(self.test_user.username))
         assert 'mediagoblin/user_pages/user.html' in context
 
+    def test_default_upload_limits(self):
+        self.user_upload_limits(uploaded=500)
+
+        # User uploaded should be 500
+        assert self.test_user.uploaded == 500
+
+        response, context = self.do_post({'title': u'Normal upload 4'},
+                                         do_follow=True,
+                                         **self.upload_data(GOOD_JPG))
+        self.check_url(response, '/u/{0}/'.format(self.test_user.username))
+        assert 'mediagoblin/user_pages/user.html' in context
+
+        # Reload user
+        self.test_user = User.query.filter_by(
+            username=self.test_user.username
+        ).first()
+
+        # Shouldn't have uploaded
+        assert self.test_user.uploaded == 500
+
+    def test_user_upload_limit(self):
+        self.user_upload_limits(uploaded=25, upload_limit=25)
+
+        # User uploaded should be 25
+        assert self.test_user.uploaded == 25
+
+        response, context = self.do_post({'title': u'Normal upload 5'},
+                                         do_follow=True,
+                                         **self.upload_data(GOOD_JPG))
+        self.check_url(response, '/u/{0}/'.format(self.test_user.username))
+        assert 'mediagoblin/user_pages/user.html' in context
+
+        # Reload user
+        self.test_user = User.query.filter_by(
+            username=self.test_user.username
+        ).first()
+
+        # Shouldn't have uploaded
+        assert self.test_user.uploaded == 25
+
+    def test_user_under_limit(self):
+        self.user_upload_limits(uploaded=499)
+
+        # User uploaded should be 499
+        assert self.test_user.uploaded == 499
+
+        response, context = self.do_post({'title': u'Normal upload 6'},
+                                         do_follow=False,
+                                         **self.upload_data(MED_PNG))
+        form = context['mediagoblin/submit/start.html']['submit_form']
+        assert form.file.errors == [u'Sorry, uploading this file will put you'
+                                    ' over your upload limit.']
+
+        # Reload user
+        self.test_user = User.query.filter_by(
+            username=self.test_user.username
+        ).first()
+
+        # Shouldn't have uploaded
+        assert self.test_user.uploaded == 499
+
+    def test_big_file(self):
+        response, context = self.do_post({'title': u'Normal upload 7'},
+                                         do_follow=False,
+                                         **self.upload_data(BIG_PNG))
+
+        form = context['mediagoblin/submit/start.html']['submit_form']
+        assert form.file.errors == [u'Sorry, the file size is too big.']
+
     def check_media(self, request, find_data, count=None):
         media = MediaEntry.query.filter_by(**find_data)
         if count is not None:
@@ -155,6 +254,7 @@ class TestSubmission:
                     'ffffffffffffffffffffffffffuuuuuuuuuuuuuuuuuuuuuuuuuu']
 
     def test_delete(self):
+        self.user_upload_limits(uploaded=50)
         response, request = self.do_post({'title': u'Balanced Goblin'},
                                          *REQUEST_CONTEXT, do_follow=True,
                                          **self.upload_data(GOOD_JPG))
@@ -199,6 +299,14 @@ class TestSubmission:
         self.check_media(request, {'id': media_id}, 0)
         self.check_comments(request, media_id, 0)
 
+        # Reload user
+        self.test_user = User.query.filter_by(
+            username = self.test_user.username
+        ).first()
+
+        # Check that user.uploaded is the same as before the upload
+        assert self.test_user.uploaded == 50
+
     def test_evil_file(self):
         # Test non-suppoerted file with non-supported extension
         # -----------------------------------------------------

mediagoblin/tests/test_submission/COPYING.txt

@@ -0,0 +1,5 @@
+Images located in this directory tree are released under a GPLv3 license
+and CC BY-SA 3.0 license. To the extent possible under law, the author(s)
+have dedicated all copyright and related and neighboring rights to these
+files to the public domain worldwide. These files are distributed without
+any warranty.

mediagoblin/user_pages/views.py

@@ -296,6 +296,11 @@ def media_confirm_delete(request, media):
     if request.method == 'POST' and form.validate():
         if form.confirm.data is True:
             username = media.get_uploader.username
+
+            media.get_uploader.uploaded = media.get_uploader.uploaded - \
+                media.file_size
+            media.get_uploader.save()
+
             # Delete MediaEntry and all related files, comments etc.
             media.delete()
             messages.add_message(