From 11e5b19701a0bc7e5fd5b39c1e28d9e6b81ad4b0 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 06:51:07 -0500 Subject: [PATCH 1/8] Moving wiping to a clean slate beneath running server/ test suite --- docs/hackinghowto.rst | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index fe2411bb..f7e46dae 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -95,6 +95,22 @@ changed. To do that, run:: need to do this when you've made code changes. +Running the server +================== + +Run:: + + ./bin/paster serve mediagoblin.ini --reload + + +Running the test suite +====================== + +Run:: + + ./bin/nosetests + + Wiping your environment for a clean-slate ----------------------------------------- @@ -122,22 +138,6 @@ Delete the following directories: MongoDB instance. Let us know! -Running the server -================== - -Run:: - - ./bin/paster serve mediagoblin.ini --reload - - -Running the test suite -====================== - -Run:: - - ./bin/nosetests - - Quickstart for Django programmers ================================= From 4509880386a4fc8c0d3d0e7c2176776593af8f02 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 06:54:57 -0500 Subject: [PATCH 2/8] Made a link to the script wiping tool feature request ticket --- docs/hackinghowto.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index f7e46dae..ef49fc43 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -133,9 +133,9 @@ Delete the following directories: .. YouCanHelp:: - If you're familiar with MongoDB and bash, we'd love to get a bash - script that removes all the GNU MediaGoblin data from an existing - MongoDB instance. Let us know! + If you're familiar with MongoDB and bash, we'd love to get a + `script that removes all the GNU MediaGoblin data from an existing + instance `_. Let us know! Quickstart for Django programmers From 1b734c4d67640b62d93cc41c4061555581e3f77f Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 06:56:26 -0500 Subject: [PATCH 3/8] It doesn't necessarily have to be bash. --- docs/hackinghowto.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index ef49fc43..46353886 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -133,9 +133,9 @@ Delete the following directories: .. YouCanHelp:: - If you're familiar with MongoDB and bash, we'd love to get a - `script that removes all the GNU MediaGoblin data from an existing - instance `_. Let us know! + If you're familiar with MongoDB, we'd love to get a `script that + removes all the GNU MediaGoblin data from an existing instance + `_. Let us know! Quickstart for Django programmers From 1dddd4e913b21ea9ee3bf29b32916814be563fcf Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 07:44:54 -0500 Subject: [PATCH 4/8] A completely evil environment destroying script. --- destroy_environment.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 destroy_environment.py diff --git a/destroy_environment.py b/destroy_environment.py new file mode 100755 index 00000000..bbdeffe9 --- /dev/null +++ b/destroy_environment.py @@ -0,0 +1,22 @@ +#!./bin/python + +import pymongo +import sys, os + +print "*** WARNING! ***" +print " Running this will destroy your mediagoblin database," +print " remove all your media files in user_dev/, etc." + +drop_it = raw_input( + 'Are you SURE you want to destroy your environment? (if so, type "yes")> ') + +if not drop_it == 'yes': + sys.exit(1) + +conn = pymongo.Connection() +conn.drop_database('mediagoblin') + +os.popen('rm -rf user_dev/media') +os.popen('rm -rf user_dev/beaker') + +print "removed all your stuff! okay, now re-run ./bin/buildout" From fa7f9c6184286f2b56f353b21ffaf1e1577569a3 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 09:37:24 -0500 Subject: [PATCH 5/8] Process media! Successfully! --- mediagoblin/models.py | 9 +++++++-- mediagoblin/process_media/__init__.py | 25 ++++++++++++++++++++----- mediagoblin/submit/views.py | 7 +++++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/mediagoblin/models.py b/mediagoblin/models.py index eef59ed4..cd6a28cc 100644 --- a/mediagoblin/models.py +++ b/mediagoblin/models.py @@ -73,11 +73,16 @@ class MediaEntry(Document): 'tags': [unicode], 'state': unicode, + # For now let's assume there can only be one main file queued + # at a time + 'queued_media_file': [unicode], + + # A dictionary of logical names to filepaths + 'media_files': dict, + # The following should be lists of lists, in appropriate file # record form - 'media_files': list, 'attachment_files': list, - 'queue_files': list, # This one should just be a single file record 'thumbnail_file': [unicode]} diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 0d02a13f..69177fee 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -29,11 +29,11 @@ def process_media_initial(media_id): entry = database.MediaEntry.one( {'_id': mongokit.ObjectId(media_id)}) - queued_filepath = entry['queue_files'].pop() + queued_filepath = entry['queued_media_file'] queued_file = queue_store.get_file(queued_filepath, 'r') with queued_file: - thumb = Image(queued_file) + thumb = Image.open(queued_file) thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS) thumb_filepath = public_store.get_unique_filepath( @@ -44,7 +44,22 @@ def process_media_initial(media_id): with public_store.get_file(thumb_filepath, 'w') as thumb_file: thumb.save(thumb_file, "JPEG") - queue_store.delete(queued_filepath) - entry.setdefault('media_files', []).append(thumb_filepath) + # we have to re-read because unlike PIL, not everything reads + # things in string representation :) + queued_file = queue_store.get_file(queued_filepath, 'rb') + + with queued_file: + main_filepath = public_store.get_unique_filepath( + ['media_entries', + unicode(entry['_id']), + queued_filepath[-1]]) + + with public_store.get_file(main_filepath, 'wb') as main_file: + main_file.write(queued_file.read()) + + queue_store.delete_file(queued_filepath) + media_files_dict = entry.setdefault('media_files', {}) + media_files_dict['thumb'] = thumb_filepath + media_files_dict['main'] = main_filepath entry.state = 'processed' - entry.save() + entry.save(validate=False) diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 926c7011..9c4eb3a4 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -22,6 +22,7 @@ from werkzeug.utils import secure_filename from mediagoblin.decorators import require_active_login from mediagoblin.submit import forms as submit_forms +from mediagoblin.process_media import process_media_initial @require_active_login @@ -52,7 +53,6 @@ def submit_start(request): # Now store generate the queueing related filename queue_filepath = request.app.queue_store.get_unique_filepath( ['media_entries', - unicode(request.user['_id']), unicode(entry['_id']), secure_filename(request.POST['file'].filename)]) @@ -64,9 +64,12 @@ def submit_start(request): queue_file.write(request.POST['file'].file.read()) # Add queued filename to the entry - entry.setdefault('queue_files', []).append(queue_filepath) + entry['queued_media_file'] = queue_filepath entry.save(validate=True) + # queue it for processing + process_media_initial.delay(unicode(entry['_id'])) + # redirect return exc.HTTPFound( location=request.urlgen("mediagoblin.submit.success")) From 12b6ecac0f5188f4b4e07f9fd70e0c4a4a59a5ae Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 10:01:11 -0500 Subject: [PATCH 6/8] Erk, we didn't save the state right before --- mediagoblin/process_media/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 69177fee..3c4d0ca1 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -61,5 +61,5 @@ def process_media_initial(media_id): media_files_dict = entry.setdefault('media_files', {}) media_files_dict['thumb'] = thumb_filepath media_files_dict['main'] = main_filepath - entry.state = 'processed' - entry.save(validate=False) + entry['state'] = u'processed' + entry.save() From 4c1e752a089313b11f872771f234c59d1b5b9982 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 10:01:26 -0500 Subject: [PATCH 7/8] Actually display submitted stuff on the mainpage. Crappy, but working! --- mediagoblin/templates/mediagoblin/root.html | 14 ++++++++++++++ mediagoblin/views.py | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mediagoblin/templates/mediagoblin/root.html b/mediagoblin/templates/mediagoblin/root.html index d6fffb8e..06a89f3f 100644 --- a/mediagoblin/templates/mediagoblin/root.html +++ b/mediagoblin/templates/mediagoblin/root.html @@ -36,4 +36,18 @@

{% endif %} + + {# temporarily, an "image gallery" that isn't one really ;) #} + +
+
    + {% for entry in media_entries %} +
  • + +
    • + {% endfor %} +
+
+ {% endblock %} diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 1081ce29..3728d4aa 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -22,11 +22,15 @@ import wtforms from mediagoblin import models def root_view(request): + media_entries = request.db.MediaEntry.find( + {u'state': u'processed'}) + template = request.template_env.get_template( 'mediagoblin/root.html') return Response( template.render( - {'request': request})) + {'request': request, + 'media_entries': media_entries})) class ImageSubmitForm(wtforms.Form): From 1c424df505b3c9f9cceb84a4fd0ac1867b7ed9b4 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 6 May 2011 10:06:57 -0500 Subject: [PATCH 8/8] Instructions on running celeryd --- docs/hackinghowto.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index 46353886..fdb53a25 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -103,6 +103,18 @@ Run:: ./bin/paster serve mediagoblin.ini --reload +Running celeryd +=============== + +You need to do this if you want your media to process and actually +show up. It's probably a good idea in development to have the web +server (above) running in one terminal and celeryd in another window. + +Run:: + + CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_celery ./bin/celeryd + + Running the test suite ======================