From 897354e6e4a9ba5ed36fe2ccd75c00a7b3d0b8d4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 28 Feb 2014 21:08:17 +0200 Subject: [PATCH 01/68] Make GNU MediaGoblin installable on Python 3. --- setup.py | 129 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/setup.py b/setup.py index 59f0ab8f..bf31e23b 100644 --- a/setup.py +++ b/setup.py @@ -14,17 +14,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function + from setuptools import setup, find_packages import os import re +import sys + +PY2 = sys.version_info[0] == 2 # six is not installed yet + READMEFILE = "README" VERSIONFILE = os.path.join("mediagoblin", "_version.py") VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" def get_version(): - verstrline = open(VERSIONFILE, "rt").read() + with open(VERSIONFILE, "rt") as fobj: + verstrline = fobj.read() mo = re.search(VSRE, verstrline, re.M) if mo: return mo.group(1) @@ -32,6 +39,56 @@ def get_version(): raise RuntimeError("Unable to find version string in %s." % VERSIONFILE) +py2_only_install_requires = [] +if PY2: + py2_only_install_requires.append('argparse') # only for < 2.7 + py2_only_install_requires.append('PasteScript') + # newer sqlalchemy-migrate requires pbr which BREAKS EVERYTHING AND IS + # TERRIBLE AND IS THE END OF ALL THINGS + # I'd love to remove this restriction. + py2_only_install_requires.append('sqlalchemy-migrate<0.8') + py2_only_install_requires.append('mock') # mock is in the stdlib for 3.3+ + +install_requires = [ + 'setuptools', # TODO: is this necessary + 'python-dateutil', + 'wtforms', + 'py-bcrypt', + 'pytest>=2.3.1', + 'pytest-xdist', + 'werkzeug>=0.7', + 'celery>=3.0', + 'kombu', + 'jinja2', + 'sphinx', # TODO: is this a docs requirement? + 'Babel<1.0', + 'webtest<2', + 'ConfigObj', + 'Markdown', + 'sqlalchemy<0.9.0, >0.8.0', + 'itsdangerous', + 'pytz', + # PLEASE change this when we can; a dependency is forcing us to set this + # specific number and it is breaking setup.py develop + 'six==1.5.2', + 'oauthlib==0.5.0', + 'unidecode', + 'ExifRead', + # Annoying. Please remove once we can! We only indirectly + # use pbr, and currently it breaks things, presumably till + # their next release. + 'pbr==0.5.22', + # This is optional: + # 'translitcodec', + # For now we're expecting that users will install this from + # their package managers. + # 'lxml', + # 'PIL', # TODO: switch to Pillow? +] + py2_only_install_requires + +with open(READMEFILE) as fobj: + long_description = fobj.read() + try: setup( name="mediagoblin", @@ -40,54 +97,8 @@ try: zip_safe=False, include_package_data = True, # scripts and dependencies - install_requires=[ - 'setuptools', - 'python-dateutil', - 'PasteScript', - 'wtforms', - 'py-bcrypt', - 'pytest>=2.3.1', - 'pytest-xdist', - 'werkzeug>=0.7', - 'celery>=3.0', - 'kombu', - 'jinja2', - 'sphinx', - 'Babel>=1.0', - 'argparse', - 'webtest<2', - 'ConfigObj', - 'Markdown', - 'sqlalchemy<0.9.0, >0.8.0', - # newer sqlalchemy-migrate requires pbr which BREAKS EVERYTHING AND IS - # TERRIBLE AND IS THE END OF ALL THINGS - # I'd love to remove this restriction. - 'sqlalchemy-migrate<0.8', - 'mock', - 'itsdangerous', - 'pytz', - 'oauthlib==0.5.0', - 'unidecode', - 'ExifRead', - - # PLEASE change this when we can; a dependency is forcing us to set this - # specific number and it is breaking setup.py develop - 'six==1.5.2' - - ## Annoying. Please remove once we can! We only indirectly - ## use pbr, and currently it breaks things, presumably till - ## their next release. - # 'pbr==0.5.22', - - ## This is optional! - # 'translitcodec', - ## For now we're expecting that users will install this from - ## their package managers. - # 'lxml', - # 'PIL', - ], - # requires=['gst'], - test_suite='nose.collector', + install_requires=install_requires, + test_suite='nose.collector', # TODO: We are using py.test now? entry_points="""\ [console_scripts] gmg = mediagoblin.gmg_commands:main_cli @@ -110,29 +121,33 @@ try: author_email='cwebber@gnu.org', url="http://mediagoblin.org/", download_url="http://mediagoblin.org/download/", - long_description=open(READMEFILE).read(), + long_description=long_description, classifiers=[ "Development Status :: 3 - Alpha", "Environment :: Web Environment", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", "Operating System :: OS Independent", "Programming Language :: Python", + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', "Topic :: Internet :: WWW/HTTP :: Dynamic Content" ], ) -except TypeError, e: +except TypeError as e: + import sys + # Check if the problem is caused by the sqlalchemy/setuptools conflict msg_as_str = str(e) if not (msg_as_str == 'dist must be a Distribution instance'): raise # If so, tell the user it is OK to just run the script again. - print "\n\n---------- NOTE ----------" - print "The setup.py command you ran failed." - print "" - print ("It is a known possible failure. Just run it again. It works the " - "second time.") - import sys + print("\n\n---------- NOTE ----------", file=sys.stderr) + print("The setup.py command you ran failed.\n", file=sys.stderr) + print("It is a known possible failure. Just run it again. It works the " + "second time.", file=sys.stderr) sys.exit(1) From 353f59350064a2ebde60c08903add30222c40616 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 28 Feb 2014 21:39:51 +0200 Subject: [PATCH 02/68] Ignore *.egg/ directories. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2524739f..6c138218 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ /celery.db /kombu.db /server-log.txt +*.egg/ # pyconfigure/automake generated files /Makefile From f6a700e8ae15f9b3368b908bded8129eb9345d7f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 7 Mar 2014 15:19:23 +0200 Subject: [PATCH 03/68] Fix some compat issues in mediagoblin/db/models.py. --- mediagoblin/db/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index b750375d..79389002 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -18,6 +18,8 @@ TODO: indexes on foreignkeys, where useful. """ +from __future__ import print_function + import logging import datetime @@ -371,7 +373,7 @@ class MediaEntry(Base, MediaEntryMixin): # Delete all related files/attachments try: delete_media_files(self) - except OSError, error: + except OSError as error: # Returns list of files we failed to delete _log.error('No such files from the user "{1}" to delete: ' '{0}'.format(str(error), self.get_uploader)) @@ -943,7 +945,7 @@ def show_table_init(engine_uri): if __name__ == '__main__': from sys import argv - print repr(argv) + print(repr(argv)) if len(argv) == 2: uri = argv[1] else: From 03ff865c429f9083ab50fdb3bda0f5bcaa91cefb Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 7 Mar 2014 15:19:53 +0200 Subject: [PATCH 04/68] Update install_requires. * Add PasteDeploy * Upgrade Babel --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf31e23b..658cf8c3 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ install_requires = [ 'kombu', 'jinja2', 'sphinx', # TODO: is this a docs requirement? - 'Babel<1.0', + 'Babel<1.0', # TODO: why <1.0 or 0.9.6? 'webtest<2', 'ConfigObj', 'Markdown', @@ -78,6 +78,7 @@ install_requires = [ # use pbr, and currently it breaks things, presumably till # their next release. 'pbr==0.5.22', + 'PasteDeploy', # This is optional: # 'translitcodec', # For now we're expecting that users will install this from From 74e77c36888b9fbec7e4f8f2b0f1d0e0487ed6b4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 7 Mar 2014 15:20:16 +0200 Subject: [PATCH 05/68] Add mediagoblin._compat module. --- mediagoblin/_compat.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 mediagoblin/_compat.py diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py new file mode 100644 index 00000000..bbd91c3f --- /dev/null +++ b/mediagoblin/_compat.py @@ -0,0 +1,3 @@ +import sys + +PY3 = sys.version_info[0] >= 3 From 7f342c72f64594775bac1bcce81b32ae9e18e6ac Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 19 Mar 2014 16:44:34 +0200 Subject: [PATCH 06/68] Continue to port GMG codebase. --- mediagoblin/_compat.py | 9 ++++++++- mediagoblin/db/migration_tools.py | 4 +++- mediagoblin/db/open.py | 3 ++- mediagoblin/media_types/image/processing.py | 5 +++-- mediagoblin/media_types/pdf/processing.py | 2 +- mediagoblin/storage/__init__.py | 4 +++- mediagoblin/storage/filestorage.py | 3 ++- mediagoblin/tools/crypto.py | 4 ++-- mediagoblin/tools/mail.py | 20 +++++++++++--------- 9 files changed, 35 insertions(+), 19 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index bbd91c3f..38c71524 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -1,3 +1,10 @@ import sys -PY3 = sys.version_info[0] >= 3 +from six import PY3, iteritems + +if PY3: + from email.mime.text import MIMEText + from urllib import parse as urlparse +else: + from email.MIMEText import MIMEText + import urlparse diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index e39070c3..e725f565 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import unicode_literals + from mediagoblin.tools.common import simple_printer from sqlalchemy import Table from sqlalchemy.sql import select @@ -39,7 +41,7 @@ class MigrationManager(object): - migration_registry: where we should find all migrations to run """ - self.name = unicode(name) + self.name = name self.models = models self.foundations = foundations self.session = session diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index 4ff0945f..9cf9e578 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -20,6 +20,7 @@ import logging from mediagoblin.db.base import Base, Session from mediagoblin import mg_globals +from mediagoblin._compat import iteritems _log = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class DatabaseMaster(object): def __init__(self, engine): self.engine = engine - for k, v in Base._decl_class_registry.iteritems(): + for k, v in iteritems(Base._decl_class_registry): setattr(self, k, v) def commit(self): diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 1db82ee7..a9b966ff 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function + try: from PIL import Image except ImportError: @@ -381,5 +383,4 @@ if __name__ == '__main__': clean = clean_exif(result) useful = get_useful(clean) - print pp.pprint( - clean) + print(pp.pprint(clean)) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index a000007a..0a173cf7 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -138,7 +138,7 @@ def is_unoconv_working(): try: proc = Popen([unoconv, '--show'], stderr=PIPE) output = proc.stderr.read() - except OSError, e: + except OSError: _log.warn(_('unoconv failing to run, check log file')) return False if 'ERROR' in output: diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 51b46c07..3aa1f4a4 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import absolute_import + import shutil import uuid @@ -268,4 +270,4 @@ def storage_system_from_config(config_section): storage_class = common.import_component(storage_class) return storage_class(**config_params) -import filestorage +from . import filestorage diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 29b8383b..404d24c3 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -21,7 +21,8 @@ from mediagoblin.storage import ( import os import shutil -import urlparse + +from mediagoblin._compat import urlparse class BasicFileStorage(StorageInterface): diff --git a/mediagoblin/tools/crypto.py b/mediagoblin/tools/crypto.py index 917e674c..14a1a72d 100644 --- a/mediagoblin/tools/crypto.py +++ b/mediagoblin/tools/crypto.py @@ -52,7 +52,7 @@ def load_key(filename): def create_key(key_dir, key_filepath): global __itsda_secret - old_umask = os.umask(077) + old_umask = os.umask(0o77) key_file = None try: if not os.path.isdir(key_dir): @@ -80,7 +80,7 @@ def setup_crypto(): key_filepath = os.path.join(key_dir, 'itsdangeroussecret.bin') try: load_key(key_filepath) - except IOError, error: + except IOError as error: if error.errno != errno.ENOENT: raise create_key(key_dir, key_filepath) diff --git a/mediagoblin/tools/mail.py b/mediagoblin/tools/mail.py index 0fabc5a9..ad2e5a19 100644 --- a/mediagoblin/tools/mail.py +++ b/mediagoblin/tools/mail.py @@ -14,9 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function, unicode_literals + import smtplib -from email.MIMEText import MIMEText from mediagoblin import mg_globals, messages +from mediagoblin._compat import MIMEText from mediagoblin.tools import common ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -119,12 +121,12 @@ def send_email(from_addr, to_addrs, subject, message_body): EMAIL_TEST_INBOX.append(message) elif mg_globals.app_config['email_debug_mode']: - print u"===== Email =====" - print u"From address: %s" % message['From'] - print u"To addresses: %s" % message['To'] - print u"Subject: %s" % message['Subject'] - print u"-- Body: --" - print message.get_payload(decode=True) + print("===== Email =====") + print("From address: %s" % message['From']) + print("To addresses: %s" % message['To']) + print("Subject: %s" % message['Subject']) + print("-- Body: --") + print(message.get_payload(decode=True)) return mhost.sendmail(from_addr, to_addrs, message.as_string()) @@ -151,5 +153,5 @@ def email_debug_message(request): if mg_globals.app_config['email_debug_mode']: # DEBUG message, no need to translate messages.add_message(request, messages.DEBUG, - u"This instance is running in email debug mode. " - u"The email will be on the console of the server process.") + "This instance is running in email debug mode. " + "The email will be on the console of the server process.") From 386c9c7c55147b27c258edd5deee36594553493d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 26 May 2014 19:50:38 +0300 Subject: [PATCH 07/68] Use six.iteritems() instead of dict.iteritems(). --- mediagoblin/db/models.py | 6 ++++-- mediagoblin/gmg_commands/__init__.py | 4 +++- mediagoblin/init/celery/__init__.py | 6 ++++-- mediagoblin/mg_globals.py | 3 ++- mediagoblin/moderation/tools.py | 7 +++++-- mediagoblin/plugins/ldap/tools.py | 4 +++- mediagoblin/plugins/piwigo/tools.py | 2 +- mediagoblin/storage/__init__.py | 4 +++- mediagoblin/storage/mountstorage.py | 4 +++- mediagoblin/tests/test_sql_migrations.py | 4 +++- mediagoblin/tests/tools.py | 5 +++-- mediagoblin/tools/exif.py | 6 ++++-- mediagoblin/tools/response.py | 3 ++- mediagoblin/tools/staticdirect.py | 4 +++- 14 files changed, 43 insertions(+), 19 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 79389002..1fecbaaa 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -40,6 +40,8 @@ from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, \ from mediagoblin.tools.files import delete_media_files from mediagoblin.tools.common import import_component +import six + # It's actually kind of annoying how sqlalchemy-migrate does this, if # I understand it right, but whatever. Anyway, don't remove this :P # @@ -319,7 +321,7 @@ class MediaEntry(Base, MediaEntryMixin): file_metadata = media_file.file_metadata or {} - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): file_metadata[key] = value media_file.file_metadata = file_metadata @@ -344,7 +346,7 @@ class MediaEntry(Base, MediaEntryMixin): media_data.get_media_entry = self else: # Update old media data - for field, value in kwargs.iteritems(): + for field, value in six.iteritems(kwargs): setattr(media_data, field, value) @memoized_property diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 9de4130e..e0400897 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -17,6 +17,8 @@ import argparse import os +import six + from mediagoblin.tools.common import import_component @@ -90,7 +92,7 @@ def main_cli(): "otherwise mediagoblin.ini")) subparsers = parser.add_subparsers(help='sub-command help') - for command_name, command_struct in SUBCOMMAND_MAP.iteritems(): + for command_name, command_struct in six.iteritems(SUBCOMMAND_MAP): if 'help' in command_struct: subparser = subparsers.add_parser( command_name, help=command_struct['help']) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 57242bf6..ffc7ca0e 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -18,6 +18,8 @@ import os import sys import logging +import six + from celery import Celery from mediagoblin.tools.pluginapi import hook_runall @@ -45,7 +47,7 @@ def get_celery_settings_dict(app_config, global_config, celery_settings = {} # Add all celery settings from config - for key, value in celery_conf.iteritems(): + for key, value in six.iteritems(celery_conf): celery_settings[key] = value # TODO: use default result stuff here if it exists @@ -98,7 +100,7 @@ def setup_celery_from_config(app_config, global_config, __import__(settings_module) this_module = sys.modules[settings_module] - for key, value in celery_settings.iteritems(): + for key, value in six.iteritems(celery_settings): setattr(this_module, key, value) if set_environ: diff --git a/mediagoblin/mg_globals.py b/mediagoblin/mg_globals.py index 26ed66fa..7da31680 100644 --- a/mediagoblin/mg_globals.py +++ b/mediagoblin/mg_globals.py @@ -21,6 +21,7 @@ import gettext import pkg_resources import threading +import six ############################# # General mediagoblin globals @@ -64,7 +65,7 @@ def setup_globals(**kwargs): """ from mediagoblin import mg_globals - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): if not hasattr(mg_globals, key): raise AssertionError("Global %s not known" % key) setattr(mg_globals, key, value) diff --git a/mediagoblin/moderation/tools.py b/mediagoblin/moderation/tools.py index e0337536..edee42ef 100644 --- a/mediagoblin/moderation/tools.py +++ b/mediagoblin/moderation/tools.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from mediagoblin import mg_globals from mediagoblin.db.models import User, Privilege, UserBan from mediagoblin.db.base import Session @@ -22,8 +24,9 @@ from mediagoblin.tools.response import redirect from datetime import datetime from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ + def take_punitive_actions(request, form, report, user): - message_body ='' + message_body = '' # The bulk of this action is running through all of the different # punitive actions that a moderator could take. @@ -212,6 +215,6 @@ def parse_report_panel_settings(form): filters['reporter_id'] = form.reporter.data filters = dict((k, v) - for k, v in filters.iteritems() if v) + for k, v in six.iteritems(filters) if v) return filters diff --git a/mediagoblin/plugins/ldap/tools.py b/mediagoblin/plugins/ldap/tools.py index 1c436792..2be2dcd7 100644 --- a/mediagoblin/plugins/ldap/tools.py +++ b/mediagoblin/plugins/ldap/tools.py @@ -16,6 +16,8 @@ import ldap import logging +import six + from mediagoblin.tools import pluginapi _log = logging.getLogger(__name__) @@ -47,7 +49,7 @@ class LDAP(object): return email def login(self, username, password): - for k, v in self.ldap_settings.iteritems(): + for k, v in six.iteritems(self.ldap_settings): try: self._connect(v) user_dn = v['LDAP_USER_DN_TEMPLATE'].format(username=username) diff --git a/mediagoblin/plugins/piwigo/tools.py b/mediagoblin/plugins/piwigo/tools.py index 484ea531..7b9b7af3 100644 --- a/mediagoblin/plugins/piwigo/tools.py +++ b/mediagoblin/plugins/piwigo/tools.py @@ -47,7 +47,7 @@ class PwgNamedArray(list): def _fill_element_dict(el, data, as_attr=()): - for k, v in data.iteritems(): + for k, v in six.iteritems(data): if k in as_attr: if not isinstance(v, six.string_types): v = str(v) diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 3aa1f4a4..12109748 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -19,6 +19,8 @@ from __future__ import absolute_import import shutil import uuid +import six + from werkzeug.utils import secure_filename from mediagoblin.tools import common @@ -259,7 +261,7 @@ def storage_system_from_config(config_section): """ # This construct is needed, because dict(config) does # not replace the variables in the config items. - config_params = dict(config_section.iteritems()) + config_params = dict(six.iteritems(config_section)) if 'storage_class' in config_params: storage_class = config_params['storage_class'] diff --git a/mediagoblin/storage/mountstorage.py b/mediagoblin/storage/mountstorage.py index dffc619b..4125a88d 100644 --- a/mediagoblin/storage/mountstorage.py +++ b/mediagoblin/storage/mountstorage.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from mediagoblin.storage import StorageInterface, clean_listy_filepath @@ -120,7 +122,7 @@ class MountStorage(StorageInterface): v = table.get(None) if v: res.append(" " * len(indent) + repr(indent) + ": " + repr(v)) - for k, v in table.iteritems(): + for k, v in six.iteritems(table): if k == None: continue res.append(" " * len(indent) + repr(k) + ":") diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index 3d67fdf6..e86dcd80 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -16,6 +16,8 @@ import copy +import six + from sqlalchemy import ( Table, Column, MetaData, Index, Integer, Float, Unicode, UnicodeText, DateTime, Boolean, @@ -190,7 +192,7 @@ def level_exits_new_table(db_conn): for level in result: - for exit_name, to_level in level['exits'].iteritems(): + for exit_name, to_level in six.iteritems(level['exits']): # Insert the level exit db_conn.execute( level_exits.insert().values( diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 060dfda9..b8e06e19 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -19,6 +19,7 @@ import os import pkg_resources import shutil +import six from paste.deploy import loadapp from webtest import TestApp @@ -142,7 +143,7 @@ def install_fixtures_simple(db, fixtures): """ Very simply install fixtures in the database """ - for collection_name, collection_fixtures in fixtures.iteritems(): + for collection_name, collection_fixtures in six.iteritems(fixtures): collection = db[collection_name] for fixture in collection_fixtures: collection.insert(fixture) @@ -162,7 +163,7 @@ def assert_db_meets_expected(db, expected): {'id': 'foo', 'some_field': 'some_value'},]} """ - for collection_name, collection_data in expected.iteritems(): + for collection_name, collection_data in six.iteritems(expected): collection = db[collection_name] for expected_document in collection_data: document = collection.query.filter_by(id=expected_document['id']).first() diff --git a/mediagoblin/tools/exif.py b/mediagoblin/tools/exif.py index 50f1aabf..736bae7d 100644 --- a/mediagoblin/tools/exif.py +++ b/mediagoblin/tools/exif.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from exifread import process_file from exifread.utils import Ratio @@ -94,7 +96,7 @@ def clean_exif(exif): 'Thumbnail JPEGInterchangeFormat'] return dict((key, _ifd_tag_to_dict(value)) for (key, value) - in exif.iteritems() if key not in disabled_tags) + in six.iteritems(exif) if key not in disabled_tags) def _ifd_tag_to_dict(tag): @@ -149,7 +151,7 @@ def get_gps_data(tags): 'latitude': tags['GPS GPSLatitude'], 'longitude': tags['GPS GPSLongitude']} - for key, dat in dms_data.iteritems(): + for key, dat in six.iteritems(dms_data): gps_data[key] = ( lambda v: float(v[0].num) / float(v[0].den) \ diff --git a/mediagoblin/tools/response.py b/mediagoblin/tools/response.py index cd99a230..1380633f 100644 --- a/mediagoblin/tools/response.py +++ b/mediagoblin/tools/response.py @@ -16,6 +16,7 @@ import json +import six import werkzeug.utils from werkzeug.wrappers import Response as wz_Response from mediagoblin.tools.template import render_template @@ -152,7 +153,7 @@ def json_response(serializable, _disable_cors=False, *args, **kw): 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'POST, GET, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type, X-Requested-With'} - for key, value in cors_headers.iteritems(): + for key, value in six.iteritems(cors_headers): response.headers.set(key, value) return response diff --git a/mediagoblin/tools/staticdirect.py b/mediagoblin/tools/staticdirect.py index 8381b8b6..881dd20e 100644 --- a/mediagoblin/tools/staticdirect.py +++ b/mediagoblin/tools/staticdirect.py @@ -24,6 +24,8 @@ import logging +import six + _log = logging.getLogger(__name__) @@ -48,7 +50,7 @@ class StaticDirect(object): def __init__(self, domains): self.domains = dict( [(key, value.rstrip('/')) - for key, value in domains.iteritems()]) + for key, value in six.iteritems(domains)]) self.cache = {} def __call__(self, filepath, domain=None): From fd19da346b7bcbc6310a4f53d3d224797a319e9c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 26 May 2014 19:52:18 +0300 Subject: [PATCH 08/68] Use six.moves.urllib.parse instead of the urlparse module. --- mediagoblin/decorators.py | 3 ++- mediagoblin/plugins/api/tools.py | 4 +++- mediagoblin/plugins/oauth/forms.py | 2 +- mediagoblin/tests/test_auth.py | 4 +++- mediagoblin/tests/test_basic_auth.py | 3 ++- mediagoblin/tests/test_ldap.py | 4 +++- mediagoblin/tests/test_notifications.py | 2 +- mediagoblin/tests/test_persona.py | 4 +++- mediagoblin/tests/test_submission.py | 3 ++- mediagoblin/webfinger/views.py | 2 +- 10 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index 8515d091..61d078b2 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -16,10 +16,11 @@ from functools import wraps -from urlparse import urljoin from werkzeug.exceptions import Forbidden, NotFound from oauthlib.oauth1 import ResourceEndpoint +from six.moves.urllib.parse import urljoin + from mediagoblin import mg_globals as mgg from mediagoblin import messages from mediagoblin.db.models import MediaEntry, User, MediaComment diff --git a/mediagoblin/plugins/api/tools.py b/mediagoblin/plugins/api/tools.py index d1b3ebb1..56256236 100644 --- a/mediagoblin/plugins/api/tools.py +++ b/mediagoblin/plugins/api/tools.py @@ -18,9 +18,11 @@ import logging import json from functools import wraps -from urlparse import urljoin from werkzeug.exceptions import Forbidden from werkzeug.wrappers import Response + +from six.moves.urllib.parse import urljoin + from mediagoblin import mg_globals from mediagoblin.tools.pluginapi import PluginManager from mediagoblin.storage.filestorage import BasicFileStorage diff --git a/mediagoblin/plugins/oauth/forms.py b/mediagoblin/plugins/oauth/forms.py index 5edd992a..d271359a 100644 --- a/mediagoblin/plugins/oauth/forms.py +++ b/mediagoblin/plugins/oauth/forms.py @@ -16,7 +16,7 @@ import wtforms -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from mediagoblin.tools.extlib.wtf_html5 import URLField from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index 1bbc3d01..e8fd76d8 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -14,10 +14,12 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse + import pkg_resources import pytest +import six.moves.urllib.parse as urlparse + from mediagoblin import mg_globals from mediagoblin.db.models import User from mediagoblin.tests.tools import get_app, fixture_add_user diff --git a/mediagoblin/tests/test_basic_auth.py b/mediagoblin/tests/test_basic_auth.py index 828f0515..e7157bee 100644 --- a/mediagoblin/tests/test_basic_auth.py +++ b/mediagoblin/tests/test_basic_auth.py @@ -13,7 +13,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse + +import six.moves.urllib.parse as urlparse from mediagoblin.db.models import User from mediagoblin.plugins.basic_auth import tools as auth_tools diff --git a/mediagoblin/tests/test_ldap.py b/mediagoblin/tests/test_ldap.py index 48efb4b6..33cf85d6 100644 --- a/mediagoblin/tests/test_ldap.py +++ b/mediagoblin/tests/test_ldap.py @@ -13,11 +13,13 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse + import pkg_resources import pytest import mock +import six.moves.urllib.parse as urlparse + from mediagoblin import mg_globals from mediagoblin.db.base import Session from mediagoblin.tests.tools import get_app diff --git a/mediagoblin/tests/test_notifications.py b/mediagoblin/tests/test_notifications.py index 3bf36f5f..37d61c41 100644 --- a/mediagoblin/tests/test_notifications.py +++ b/mediagoblin/tests/test_notifications.py @@ -16,7 +16,7 @@ import pytest -import urlparse +import six.moves.urllib.parse as urlparse from mediagoblin.tools import template, mail diff --git a/mediagoblin/tests/test_persona.py b/mediagoblin/tests/test_persona.py index a1cd30eb..dbc4797a 100644 --- a/mediagoblin/tests/test_persona.py +++ b/mediagoblin/tests/test_persona.py @@ -13,11 +13,13 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse + import pkg_resources import pytest import mock +import six.moves.urllib.parse as urlparse + pytest.importorskip("requests") from mediagoblin import mg_globals diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index b5b13ed3..5ddd882c 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -18,10 +18,11 @@ import sys reload(sys) sys.setdefaultencoding('utf-8') -import urlparse import os import pytest +import six.moves.urllib.parse as urlparse + from mediagoblin.tests.tools import fixture_add_user from mediagoblin import mg_globals from mediagoblin.db.models import MediaEntry, User diff --git a/mediagoblin/webfinger/views.py b/mediagoblin/webfinger/views.py index 97fc3ef7..c4dfedf2 100644 --- a/mediagoblin/webfinger/views.py +++ b/mediagoblin/webfinger/views.py @@ -19,7 +19,7 @@ For references, see docstring in mediagoblin/webfinger/__init__.py import re -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from mediagoblin.tools.response import render_to_response, render_404 From a80c74bbcc6ac0208cfef725a441810a29876cff Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 26 May 2014 19:53:10 +0300 Subject: [PATCH 09/68] Add ugettext and ungettext helpers to mediagoblin._compat. --- mediagoblin/_compat.py | 6 ++++++ mediagoblin/tools/template.py | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index 38c71524..a6b775fc 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -2,9 +2,15 @@ import sys from six import PY3, iteritems +from mediagoblin import mg_globals + if PY3: from email.mime.text import MIMEText from urllib import parse as urlparse + ugettext = mg_globals.thread_scope.translations.gettext + ungettext = mg_globals.thread_scope.translations.ngettext else: from email.MIMEText import MIMEText import urlparse + ugettext = mg_globals.thread_scope.translations.ugettext + ungettext = mg_globals.thread_scope.translations.ungettext diff --git a/mediagoblin/tools/template.py b/mediagoblin/tools/template.py index e5acdf45..359ddd38 100644 --- a/mediagoblin/tools/template.py +++ b/mediagoblin/tools/template.py @@ -33,6 +33,7 @@ from mediagoblin.tools.pluginapi import get_hook_templates, hook_transform from mediagoblin.tools.timesince import timesince from mediagoblin.meddleware.csrf import render_csrf_form_token +from mediagoblin._compat import ugettext, ungettext SETUP_JINJA_ENVS = {} @@ -66,9 +67,7 @@ def get_jinja_env(template_loader, locale): 'jinja2.ext.i18n', 'jinja2.ext.autoescape', TemplateHookExtension] + local_exts) - template_env.install_gettext_callables( - mg_globals.thread_scope.translations.ugettext, - mg_globals.thread_scope.translations.ungettext) + template_env.install_gettext_callables(ugettext, ungettext) # All templates will know how to ... # ... fetch all waiting messages and remove them from the queue From e49b7e02b2150413d2e78db709277fae0887be46 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 2 Jun 2014 20:59:28 +0300 Subject: [PATCH 10/68] Use six.text_type instead of unicode(). I will be switch to use ``from __future__ import unicode_literals`` later. --- mediagoblin/auth/tools.py | 4 +++- mediagoblin/auth/views.py | 4 +++- mediagoblin/db/migrations.py | 6 ++++-- mediagoblin/db/models.py | 4 ++-- mediagoblin/edit/views.py | 16 ++++++++------- mediagoblin/gmg_commands/addmedia.py | 16 +++++++++------ mediagoblin/gmg_commands/users.py | 10 ++++++---- mediagoblin/media_types/ascii/processing.py | 4 +++- mediagoblin/media_types/blog/views.py | 22 +++++++++++---------- mediagoblin/media_types/image/processing.py | 6 ++++-- mediagoblin/plugins/api/views.py | 12 ++++++----- mediagoblin/plugins/basic_auth/tools.py | 4 +++- mediagoblin/plugins/httpapiauth/__init__.py | 4 +++- mediagoblin/plugins/ldap/views.py | 5 ++++- mediagoblin/plugins/oauth/tools.py | 6 ++++-- mediagoblin/plugins/oauth/views.py | 10 ++++++---- mediagoblin/plugins/openid/store.py | 6 ++++-- mediagoblin/plugins/openid/views.py | 5 ++++- mediagoblin/plugins/persona/views.py | 4 +++- mediagoblin/plugins/piwigo/views.py | 6 ++++-- mediagoblin/processing/__init__.py | 6 ++++-- mediagoblin/storage/__init__.py | 2 +- mediagoblin/submit/lib.py | 8 +++++--- mediagoblin/submit/views.py | 12 ++++++----- mediagoblin/tests/test_auth.py | 6 ++++-- mediagoblin/tests/test_http_callback.py | 4 +++- mediagoblin/tests/test_ldap.py | 3 ++- mediagoblin/tests/test_oauth2.py | 6 ++++-- mediagoblin/tests/test_openid.py | 3 ++- mediagoblin/tests/test_persona.py | 3 ++- mediagoblin/tests/test_reporting.py | 5 +++-- mediagoblin/tests/test_storage.py | 4 +++- mediagoblin/tests/test_submission.py | 3 ++- mediagoblin/tests/test_util.py | 6 ++++-- mediagoblin/tools/url.py | 4 +++- mediagoblin/user_pages/views.py | 6 ++++-- 36 files changed, 151 insertions(+), 84 deletions(-) diff --git a/mediagoblin/auth/tools.py b/mediagoblin/auth/tools.py index 88716e1c..8c919498 100644 --- a/mediagoblin/auth/tools.py +++ b/mediagoblin/auth/tools.py @@ -16,6 +16,8 @@ import logging + +import six import wtforms from sqlalchemy import or_ @@ -140,7 +142,7 @@ def register_user(request, register_form): user.save() # log the user in - request.session['user_id'] = unicode(user.id) + request.session['user_id'] = six.text_type(user.id) request.session.save() # send verification email diff --git a/mediagoblin/auth/views.py b/mediagoblin/auth/views.py index 3d132f84..a90db0ea 100644 --- a/mediagoblin/auth/views.py +++ b/mediagoblin/auth/views.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from itsdangerous import BadSignature from mediagoblin import messages, mg_globals @@ -93,7 +95,7 @@ def login(request): # set up login in session if login_form.stay_logged_in.data: request.session['stay_logged_in'] = True - request.session['user_id'] = unicode(user.id) + request.session['user_id'] = six.text_type(user.id) request.session.save() if request.form.get('next'): diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 426080a2..dd96c3f3 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -17,6 +17,8 @@ import datetime import uuid +import six + from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, Integer, Unicode, UnicodeText, DateTime, ForeignKey, Date) @@ -248,7 +250,7 @@ def mediaentry_new_slug_era(db): for row in db.execute(media_table.select()): # no slug, try setting to an id if not row.slug: - append_garbage_till_unique(row, unicode(row.id)) + append_garbage_till_unique(row, six.text_type(row.id)) # has "=" or ":" in it... we're getting rid of those elif u"=" in row.slug or u":" in row.slug: append_garbage_till_unique( @@ -277,7 +279,7 @@ def unique_collections_slug(db): existing_slugs[row.creator].append(row.slug) for row_id in slugs_to_change: - new_slug = unicode(uuid.uuid4()) + new_slug = six.text_type(uuid.uuid4()) db.execute(collection_table.update(). where(collection_table.c.id == row_id). values(slug=new_slug)) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 1fecbaaa..4b2fb632 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -304,7 +304,7 @@ class MediaEntry(Base, MediaEntryMixin): return the value of the key. """ media_file = MediaFile.query.filter_by(media_entry=self.id, - name=unicode(file_key)).first() + name=six.text_type(file_key)).first() if media_file: if metadata_key: @@ -317,7 +317,7 @@ class MediaEntry(Base, MediaEntryMixin): Update the file_metadata of a MediaFile. """ media_file = MediaFile.query.filter_by(media_entry=self.id, - name=unicode(file_key)).first() + name=six.text_type(file_key)).first() file_metadata = media_file.file_metadata or {} diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 80590875..52f808b6 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from datetime import datetime from itsdangerous import BadSignature @@ -77,7 +79,7 @@ def edit_media(request, media): media.tags = convert_to_tag_list_of_dicts( form.tags.data) - media.license = unicode(form.license.data) or None + media.license = six.text_type(form.license.data) or None media.slug = slug media.save() @@ -135,7 +137,7 @@ def edit_attachments(request, media): attachment_public_filepath \ = mg_globals.public_store.get_unique_filepath( - ['media_entries', unicode(media.id), 'attachment', + ['media_entries', six.text_type(media.id), 'attachment', public_filename]) attachment_public_file = mg_globals.public_store.get_file( @@ -200,8 +202,8 @@ def edit_profile(request, url_user=None): bio=user.bio) if request.method == 'POST' and form.validate(): - user.url = unicode(form.url.data) - user.bio = unicode(form.bio.data) + user.url = six.text_type(form.url.data) + user.bio = six.text_type(form.bio.data) user.save() @@ -316,9 +318,9 @@ def edit_collection(request, collection): form.slug.errors.append( _(u'A collection with that slug already exists for this user.')) else: - collection.title = unicode(form.title.data) - collection.description = unicode(form.description.data) - collection.slug = unicode(form.slug.data) + collection.title = six.text_type(form.title.data) + collection.description = six.text_type(form.description.data) + collection.slug = six.text_type(form.slug.data) collection.save() diff --git a/mediagoblin/gmg_commands/addmedia.py b/mediagoblin/gmg_commands/addmedia.py index c33a8c56..34add88b 100644 --- a/mediagoblin/gmg_commands/addmedia.py +++ b/mediagoblin/gmg_commands/addmedia.py @@ -14,8 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function + import os +import six + from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, @@ -68,14 +72,14 @@ def addmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print("Sorry, no user by username '%s'" % args.username) return # check for the file, if it exists... filename = os.path.split(args.filename)[-1] abs_filename = os.path.abspath(args.filename) if not os.path.exists(abs_filename): - print "Can't find a file with filename '%s'" % args.filename + print("Can't find a file with filename '%s'" % args.filename) return upload_limit, max_file_size = get_upload_file_limits(user) @@ -85,7 +89,7 @@ def addmedia(args): if some_string is None: return None else: - return unicode(some_string) + return six.text_type(some_string) try: submit_media( @@ -98,8 +102,8 @@ def addmedia(args): tags_string=maybe_unicodeify(args.tags) or u"", upload_limit=upload_limit, max_file_size=max_file_size) except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print("This file is larger than the upload limits for this site.") except UserUploadLimit: - print "This file will put this user past their upload limits." + print("This file will put this user past their upload limits.") except UserPastUploadLimit: - print "This user is already past their upload limits." + print("This user is already past their upload limits.") diff --git a/mediagoblin/gmg_commands/users.py b/mediagoblin/gmg_commands/users.py index 4a730d9e..9014da87 100644 --- a/mediagoblin/gmg_commands/users.py +++ b/mediagoblin/gmg_commands/users.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from mediagoblin.gmg_commands import util as commands_util from mediagoblin import auth from mediagoblin import mg_globals @@ -50,8 +52,8 @@ def adduser(args): else: # Create the user entry = db.User() - entry.username = unicode(args.username.lower()) - entry.email = unicode(args.email) + entry.username = six.text_type(args.username.lower()) + entry.email = six.text_type(args.email) entry.pw_hash = auth.gen_password_hash(args.password) default_privileges = [ db.Privilege.query.filter( @@ -81,7 +83,7 @@ def makeadmin(args): db = mg_globals.database user = db.User.query.filter_by( - username=unicode(args.username.lower())).one() + username=six.text_type(args.username.lower())).one() if user: user.all_privileges.append( db.Privilege.query.filter( @@ -108,7 +110,7 @@ def changepw(args): db = mg_globals.database user = db.User.query.filter_by( - username=unicode(args.username.lower())).one() + username=six.text_type(args.username.lower())).one() if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 84030362..6f388573 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -22,6 +22,8 @@ except ImportError: import Image import logging +import six + from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( create_pub_filepath, FilenameBuilder, @@ -104,7 +106,7 @@ class CommonAsciiProcessor(MediaProcessor): # Encode the unicode instance to ASCII and replace any # non-ASCII with an HTML entity (&# unicode_file.write( - unicode(orig_file.read().decode( + six.text_type(orig_file.read().decode( self.charset)).encode( 'ascii', 'xmlcharrefreplace')) diff --git a/mediagoblin/media_types/blog/views.py b/mediagoblin/media_types/blog/views.py index a367bef8..3a4dfe6a 100644 --- a/mediagoblin/media_types/blog/views.py +++ b/mediagoblin/media_types/blog/views.py @@ -19,6 +19,8 @@ _log = logging.getLogger(__name__) from datetime import datetime +import six + from werkzeug.exceptions import Forbidden from mediagoblin.tools import pluginapi @@ -75,8 +77,8 @@ def blog_edit(request): if request.method=='POST' and form.validate(): _log.info("Here") blog = request.db.Blog() - blog.title = unicode(form.title.data) - blog.description = unicode(cleaned_markdown_conversion((form.description.data))) + blog.title = six.text_type(form.title.data) + blog.description = six.text_type(cleaned_markdown_conversion((form.description.data))) blog.author = request.user.id blog.generate_slug() @@ -112,8 +114,8 @@ def blog_edit(request): 'app_config': mg_globals.app_config}) else: if request.method == 'POST' and form.validate(): - blog.title = unicode(form.title.data) - blog.description = unicode(cleaned_markdown_conversion((form.description.data))) + blog.title = six.text_type(form.title.data) + blog.description = six.text_type(cleaned_markdown_conversion((form.description.data))) blog.author = request.user.id blog.generate_slug() @@ -137,10 +139,10 @@ def blogpost_create(request): blogpost = request.db.MediaEntry() blogpost.media_type = 'mediagoblin.media_types.blogpost' - blogpost.title = unicode(form.title.data) - blogpost.description = unicode(cleaned_markdown_conversion((form.description.data))) + blogpost.title = six.text_type(form.title.data) + blogpost.description = six.text_type(cleaned_markdown_conversion((form.description.data))) blogpost.tags = convert_to_tag_list_of_dicts(form.tags.data) - blogpost.license = unicode(form.license.data) or None + blogpost.license = six.text_type(form.license.data) or None blogpost.uploader = request.user.id blogpost.generate_slug() @@ -187,10 +189,10 @@ def blogpost_edit(request): form = blog_forms.BlogPostEditForm(request.form, **defaults) if request.method == 'POST' and form.validate(): - blogpost.title = unicode(form.title.data) - blogpost.description = unicode(cleaned_markdown_conversion((form.description.data))) + blogpost.title = six.text_type(form.title.data) + blogpost.description = six.text_type(cleaned_markdown_conversion((form.description.data))) blogpost.tags = convert_to_tag_list_of_dicts(form.tags.data) - blogpost.license = unicode(form.license.data) + blogpost.license = six.text_type(form.license.data) set_blogpost_state(request, blogpost) blogpost.generate_slug() blogpost.save() diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index a9b966ff..ae9ece24 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -24,6 +24,8 @@ import os import logging import argparse +import six + from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( BadMediaFail, FilenameBuilder, @@ -67,7 +69,7 @@ def resize_image(entry, resized, keyname, target_name, new_size, resize_filter = PIL_FILTERS[filter.upper()] except KeyError: raise Exception('Filter "{0}" not found, choose one of {1}'.format( - unicode(filter), + six.text_type(filter), u', '.join(PIL_FILTERS.keys()))) resized.thumbnail(new_size, resize_filter) @@ -116,7 +118,7 @@ def resize_tool(entry, or im.size[1] > new_size[1]\ or exif_image_needs_rotation(exif_tags): resize_image( - entry, im, unicode(keyname), target_name, + entry, im, six.text_type(keyname), target_name, tuple(new_size), exif_tags, conversions_subdir, quality, filter) diff --git a/mediagoblin/plugins/api/views.py b/mediagoblin/plugins/api/views.py index 69fae7bb..a9aaacee 100644 --- a/mediagoblin/plugins/api/views.py +++ b/mediagoblin/plugins/api/views.py @@ -17,6 +17,8 @@ import json import logging +import six + from werkzeug.exceptions import BadRequest from werkzeug.wrappers import Response @@ -55,15 +57,15 @@ def post_entry(request): callback_url = request.form.get('callback_url') if callback_url: - callback_url = unicode(callback_url) + callback_url = six.text_type(callback_url) try: entry = submit_media( mg_app=request.app, user=request.user, submitted_file=request.files['file'], filename=request.files['file'].filename, - title=unicode(request.form.get('title')), - description=unicode(request.form.get('description')), - license=unicode(request.form.get('license', '')), + title=six.text_type(request.form.get('title')), + description=six.text_type(request.form.get('description')), + license=six.text_type(request.form.get('license', '')), upload_limit=upload_limit, max_file_size=max_file_size, callback_url=callback_url) @@ -88,7 +90,7 @@ def post_entry(request): ''' if isinstance(e, InvalidFileType) or \ isinstance(e, FileTypeNotSupported): - raise BadRequest(unicode(e)) + raise BadRequest(six.text_type(e)) else: raise diff --git a/mediagoblin/plugins/basic_auth/tools.py b/mediagoblin/plugins/basic_auth/tools.py index f943bf39..13f240b2 100644 --- a/mediagoblin/plugins/basic_auth/tools.py +++ b/mediagoblin/plugins/basic_auth/tools.py @@ -16,6 +16,8 @@ import bcrypt import random +import six + from mediagoblin import mg_globals from mediagoblin.tools.crypto import get_timed_signer_url from mediagoblin.tools.mail import send_email @@ -66,7 +68,7 @@ def bcrypt_gen_password_hash(raw_pass, extra_salt=None): if extra_salt: raw_pass = u"%s:%s" % (extra_salt, raw_pass) - return unicode( + return six.text_type( bcrypt.hashpw(raw_pass.encode('utf-8'), bcrypt.gensalt())) diff --git a/mediagoblin/plugins/httpapiauth/__init__.py b/mediagoblin/plugins/httpapiauth/__init__.py index 2b2d593c..d7180463 100644 --- a/mediagoblin/plugins/httpapiauth/__init__.py +++ b/mediagoblin/plugins/httpapiauth/__init__.py @@ -16,6 +16,8 @@ import logging +import six + from werkzeug.exceptions import Unauthorized from mediagoblin.auth.tools import check_login_simple @@ -40,7 +42,7 @@ class HTTPAuth(Auth): if not request.authorization: return False - user = check_login_simple(unicode(request.authorization['username']), + user = check_login_simple(six.text_type(request.authorization['username']), request.authorization['password']) if user: diff --git a/mediagoblin/plugins/ldap/views.py b/mediagoblin/plugins/ldap/views.py index aef1bf56..be434daf 100644 --- a/mediagoblin/plugins/ldap/views.py +++ b/mediagoblin/plugins/ldap/views.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + +import six + from mediagoblin import mg_globals, messages from mediagoblin.auth.tools import register_user from mediagoblin.db.models import User @@ -40,7 +43,7 @@ def login(request): if user: # set up login in session - request.session['user_id'] = unicode(user.id) + request.session['user_id'] = six.text_type(user.id) request.session.save() if request.form.get('next'): diff --git a/mediagoblin/plugins/oauth/tools.py b/mediagoblin/plugins/oauth/tools.py index af0a3305..2053d5d4 100644 --- a/mediagoblin/plugins/oauth/tools.py +++ b/mediagoblin/plugins/oauth/tools.py @@ -23,6 +23,8 @@ from datetime import datetime from functools import wraps +import six + from mediagoblin.tools.response import json_response @@ -86,7 +88,7 @@ def create_token(client, user): def generate_identifier(): ''' Generates a ``uuid.uuid4()`` ''' - return unicode(uuid.uuid4()) + return six.text_type(uuid.uuid4()) def generate_token(): @@ -110,5 +112,5 @@ def generate_secret(): ''' # XXX: We might not want it to use bcrypt, since bcrypt takes its time to # generate the result. - return unicode(getrandbits(192)) + return six.text_type(getrandbits(192)) diff --git a/mediagoblin/plugins/oauth/views.py b/mediagoblin/plugins/oauth/views.py index de637d6b..9e3b87c9 100644 --- a/mediagoblin/plugins/oauth/views.py +++ b/mediagoblin/plugins/oauth/views.py @@ -19,6 +19,8 @@ import logging from urllib import urlencode +import six + from werkzeug.exceptions import BadRequest from mediagoblin.tools.response import render_to_response, redirect, json_response @@ -44,11 +46,11 @@ def register_client(request): if request.method == 'POST' and form.validate(): client = OAuthClient() - client.name = unicode(form.name.data) - client.description = unicode(form.description.data) - client.type = unicode(form.type.data) + client.name = six.text_type(form.name.data) + client.description = six.text_type(form.description.data) + client.type = six.text_type(form.type.data) client.owner_id = request.user.id - client.redirect_uri = unicode(form.redirect_uri.data) + client.redirect_uri = six.text_type(form.redirect_uri.data) client.save() diff --git a/mediagoblin/plugins/openid/store.py b/mediagoblin/plugins/openid/store.py index 8f9a7012..24726814 100644 --- a/mediagoblin/plugins/openid/store.py +++ b/mediagoblin/plugins/openid/store.py @@ -16,6 +16,8 @@ import base64 import time +import six + from openid.association import Association as OIDAssociation from openid.store.interface import OpenIDStore from openid.store import nonce @@ -34,12 +36,12 @@ class SQLAlchemyOpenIDStore(OpenIDStore): if not assoc: assoc = Association() - assoc.server_url = unicode(server_url) + assoc.server_url = six.text_type(server_url) assoc.handle = association.handle # django uses base64 encoding, python-openid uses a blob field for # secret - assoc.secret = unicode(base64.encodestring(association.secret)) + assoc.secret = six.text_type(base64.encodestring(association.secret)) assoc.issued = association.issued assoc.lifetime = association.lifetime assoc.assoc_type = association.assoc_type diff --git a/mediagoblin/plugins/openid/views.py b/mediagoblin/plugins/openid/views.py index bb2de7ab..71f444fa 100644 --- a/mediagoblin/plugins/openid/views.py +++ b/mediagoblin/plugins/openid/views.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + +import six + from openid.consumer import consumer from openid.consumer.discover import DiscoveryFailure from openid.extensions.sreg import SRegRequest, SRegResponse @@ -186,7 +189,7 @@ def finish_login(request): if user: # Set up login in session - request.session['user_id'] = unicode(user.id) + request.session['user_id'] = six.text_type(user.id) request.session.save() if request.session.get('next'): diff --git a/mediagoblin/plugins/persona/views.py b/mediagoblin/plugins/persona/views.py index 1bba3b8c..41d38353 100644 --- a/mediagoblin/plugins/persona/views.py +++ b/mediagoblin/plugins/persona/views.py @@ -17,6 +17,8 @@ import json import logging import requests +import six + from werkzeug.exceptions import BadRequest from mediagoblin import messages, mg_globals @@ -63,7 +65,7 @@ def login(request): user = query.user if query else None if user: - request.session['user_id'] = unicode(user.id) + request.session['user_id'] = six.text_type(user.id) request.session['persona_login_email'] = email request.session.save() diff --git a/mediagoblin/plugins/piwigo/views.py b/mediagoblin/plugins/piwigo/views.py index f913a730..1fe1e576 100644 --- a/mediagoblin/plugins/piwigo/views.py +++ b/mediagoblin/plugins/piwigo/views.py @@ -17,6 +17,8 @@ import logging import re +import six + from werkzeug.exceptions import MethodNotAllowed, BadRequest, NotImplemented from werkzeug.wrappers import BaseResponse @@ -133,8 +135,8 @@ def pwg_images_addSimple(request): mg_app=request.app, user=request.user, submitted_file=request.files['image'], filename=request.files['image'].filename, - title=unicode(form.name.data), - description=unicode(form.comment.data), + title=six.text_type(form.name.data), + description=six.text_type(form.comment.data), upload_limit=upload_limit, max_file_size=max_file_size) collection_id = form.category.data diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 102fd5de..5a88ddea 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -24,6 +24,8 @@ except: import logging import os +import six + from mediagoblin import mg_globals as mgg from mediagoblin.db.util import atomic_update from mediagoblin.db.models import MediaEntry @@ -46,7 +48,7 @@ class ProgressCallback(object): def create_pub_filepath(entry, filename): return mgg.public_store.get_unique_filepath( ['media_entries', - unicode(entry.id), + six.text_type(entry.id), filename]) @@ -319,7 +321,7 @@ def mark_entry_failed(entry_id, exc): atomic_update(mgg.database.MediaEntry, {'id': entry_id}, {u'state': u'failed', - u'fail_error': unicode(exc.exception_path), + u'fail_error': six.text_type(exc.exception_path), u'fail_metadata': exc.metadata}) else: _log.warn("No idea what happened here, but it failed: %r", exc) diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 12109748..8462e08e 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -224,7 +224,7 @@ def clean_listy_filepath(listy_filepath): A cleaned list of unicode objects. """ cleaned_filepath = [ - unicode(secure_filename(filepath)) + six.text_type(secure_filename(filepath)) for filepath in listy_filepath] if u'' in cleaned_filepath: diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index c70e2731..7d3ea8df 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -18,6 +18,8 @@ import logging import uuid from os.path import splitext +import six + from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage @@ -127,7 +129,7 @@ def submit_media(mg_app, user, submitted_file, filename, # If the filename contains non ascii generate a unique name if not all(ord(c) < 128 for c in filename): - filename = unicode(uuid.uuid4()) + splitext(filename)[-1] + filename = six.text_type(uuid.uuid4()) + splitext(filename)[-1] # Sniff the submitted media to determine which # media plugin should handle processing @@ -136,7 +138,7 @@ def submit_media(mg_app, user, submitted_file, filename, # create entry and save in database entry = new_upload_entry(user) entry.media_type = media_type - entry.title = (title or unicode(splitext(filename)[0])) + entry.title = (title or six.text_type(splitext(filename)[0])) entry.description = description or u"" @@ -210,7 +212,7 @@ def prepare_queue_task(app, entry, filename): # (If we got it off the task's auto-generation, there'd be # a risk of a race condition when we'd save after sending # off the task) - task_id = unicode(uuid.uuid4()) + task_id = six.text_type(uuid.uuid4()) entry.queued_task_id = task_id # Now store generate the queueing related filename diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 42c378a8..b0588599 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six + from mediagoblin import messages import mediagoblin.mg_globals as mg_globals @@ -59,9 +61,9 @@ def submit_start(request): mg_app=request.app, user=request.user, submitted_file=request.files['file'], filename=request.files['file'].filename, - title=unicode(submit_form.title.data), - description=unicode(submit_form.description.data), - license=unicode(submit_form.license.data) or None, + title=six.text_type(submit_form.title.data), + description=six.text_type(submit_form.description.data), + license=six.text_type(submit_form.license.data) or None, tags_string=submit_form.tags.data, upload_limit=upload_limit, max_file_size=max_file_size, urlgen=request.urlgen) @@ -117,8 +119,8 @@ def add_collection(request, media=None): if request.method == 'POST' and submit_form.validate(): collection = request.db.Collection() - collection.title = unicode(submit_form.title.data) - collection.description = unicode(submit_form.description.data) + collection.title = six.text_type(submit_form.title.data) + collection.description = six.text_type(submit_form.description.data) collection.creator = request.user.id collection.generate_slug() diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index e8fd76d8..9b7f9825 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -18,6 +18,8 @@ import pkg_resources import pytest +import six + import six.moves.urllib.parse as urlparse from mediagoblin import mg_globals @@ -109,7 +111,7 @@ def test_register_views(test_app): ## Make sure user is logged in request = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/user_pages/user_nonactive.html']['request'] - assert request.session['user_id'] == unicode(new_user.id) + assert request.session['user_id'] == six.text_type(new_user.id) ## Make sure we get email confirmation, and try verifying assert len(mail.EMAIL_TEST_INBOX) == 1 @@ -307,7 +309,7 @@ def test_authentication_views(test_app): # Make sure user is in the session context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session - assert session['user_id'] == unicode(test_user.id) + assert session['user_id'] == six.text_type(test_user.id) # Successful logout # ----------------- diff --git a/mediagoblin/tests/test_http_callback.py b/mediagoblin/tests/test_http_callback.py index 64b7ee8f..d0a8c823 100644 --- a/mediagoblin/tests/test_http_callback.py +++ b/mediagoblin/tests/test_http_callback.py @@ -17,6 +17,8 @@ import json import pytest +import six + from urlparse import urlparse, parse_qs from mediagoblin import mg_globals @@ -63,7 +65,7 @@ class TestHTTPCallback(object): code = parse_qs(urlparse(redirect.location).query)['code'][0] client = self.db.OAuthClient.query.filter( - self.db.OAuthClient.identifier == unicode(client_id)).first() + self.db.OAuthClient.identifier == six.text_type(client_id)).first() client_secret = client.secret diff --git a/mediagoblin/tests/test_ldap.py b/mediagoblin/tests/test_ldap.py index 33cf85d6..e69a3203 100644 --- a/mediagoblin/tests/test_ldap.py +++ b/mediagoblin/tests/test_ldap.py @@ -17,6 +17,7 @@ import pkg_resources import pytest import mock +import six import six.moves.urllib.parse as urlparse @@ -122,6 +123,6 @@ def test_ldap_plugin(ldap_plugin_app): # Make sure user is in the session context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session - assert session['user_id'] == unicode(test_user.id) + assert session['user_id'] == six.text_type(test_user.id) _test_authentication() diff --git a/mediagoblin/tests/test_oauth2.py b/mediagoblin/tests/test_oauth2.py index 957f4e65..6bdb729e 100644 --- a/mediagoblin/tests/test_oauth2.py +++ b/mediagoblin/tests/test_oauth2.py @@ -18,6 +18,8 @@ import json import logging import pytest +import six + from urlparse import parse_qs, urlparse from mediagoblin import mg_globals @@ -154,7 +156,7 @@ class TestOAuth(object): code = self.get_code_from_redirect_uri(code_redirect.location) client = self.db.OAuthClient.query.filter( - self.db.OAuthClient.identifier == unicode(client_id)).first() + self.db.OAuthClient.identifier == six.text_type(client_id)).first() token_res = self.test_app.get('/oauth-2/access_token?client_id={0}&\ code={1}&client_secret={2}'.format(client_id, code, client.secret)) @@ -182,7 +184,7 @@ code={1}&client_secret={2}'.format(client_id, code, client.secret)) code = self.get_code_from_redirect_uri(code_redirect.location) client = self.db.OAuthClient.query.filter( - self.db.OAuthClient.identifier == unicode(client_id)).first() + self.db.OAuthClient.identifier == six.text_type(client_id)).first() token_res = self.test_app.get('/oauth-2/access_token?\ code={0}&client_secret={1}'.format(code, client.secret)) diff --git a/mediagoblin/tests/test_openid.py b/mediagoblin/tests/test_openid.py index 0424fdda..7ef01052 100644 --- a/mediagoblin/tests/test_openid.py +++ b/mediagoblin/tests/test_openid.py @@ -18,6 +18,7 @@ import urlparse import pkg_resources import pytest import mock +import six openid_consumer = pytest.importorskip( "openid.consumer.consumer") @@ -206,7 +207,7 @@ class TestOpenIDPlugin(object): # Make sure user is in the session context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session - assert session['user_id'] == unicode(test_user.id) + assert session['user_id'] == six.text_type(test_user.id) _test_new_user() diff --git a/mediagoblin/tests/test_persona.py b/mediagoblin/tests/test_persona.py index dbc4797a..f2dd4001 100644 --- a/mediagoblin/tests/test_persona.py +++ b/mediagoblin/tests/test_persona.py @@ -17,6 +17,7 @@ import pkg_resources import pytest import mock +import six import six.moves.urllib.parse as urlparse @@ -142,7 +143,7 @@ class TestPersonaPlugin(object): # Make sure user is in the session context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] session = context['request'].session - assert session['user_id'] == unicode(test_user.id) + assert session['user_id'] == six.text_type(test_user.id) _test_registration() diff --git a/mediagoblin/tests/test_reporting.py b/mediagoblin/tests/test_reporting.py index a154a061..6a9fe205 100644 --- a/mediagoblin/tests/test_reporting.py +++ b/mediagoblin/tests/test_reporting.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import pytest +import six from mediagoblin.tools import template from mediagoblin.tests.tools import (fixture_add_user, fixture_media_entry, @@ -75,7 +76,7 @@ class TestReportFiling: response, context = self.do_post( {'report_reason':u'Testing Media Report', - 'reporter_id':unicode(allie_id)},url= media_uri_slug + "report/") + 'reporter_id':six.text_type(allie_id)},url= media_uri_slug + "report/") assert response.status == "302 FOUND" @@ -110,7 +111,7 @@ class TestReportFiling: response, context = self.do_post({ 'report_reason':u'Testing Comment Report', - 'reporter_id':unicode(allie_id)},url= comment_uri_slug + "report/") + 'reporter_id':six.text_type(allie_id)},url= comment_uri_slug + "report/") assert response.status == "302 FOUND" diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index f6f1d18f..21947415 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -19,6 +19,8 @@ import os import tempfile import pytest +import six + from werkzeug.utils import secure_filename from mediagoblin import storage @@ -78,7 +80,7 @@ def test_storage_system_from_config(): 'mediagoblin.tests.test_storage:FakeStorageSystem'}) assert this_storage.foobie == 'eiboof' assert this_storage.blech == 'hcelb' - assert unicode(this_storage.__class__) == \ + assert six.text_type(this_storage.__class__) == \ u'mediagoblin.tests.test_storage.FakeStorageSystem' diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index 5ddd882c..6143c0ac 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -20,6 +20,7 @@ sys.setdefaultencoding('utf-8') import os import pytest +import six import six.moves.urllib.parse as urlparse @@ -35,7 +36,7 @@ from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG GOOD_TAG_STRING = u'yin,yang' -BAD_TAG_STRING = unicode('rage,' + 'f' * 26 + 'u' * 26) +BAD_TAG_STRING = six.text_type('rage,' + 'f' * 26 + 'u' * 26) FORM_CONTEXT = ['mediagoblin/submit/start.html', 'submit_form'] REQUEST_CONTEXT = ['mediagoblin/user_pages/user.html', 'request'] diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index 9d9b1c16..e108e9bb 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -16,6 +16,8 @@ import email +import six + from mediagoblin.tools import common, url, translate, mail, text, testing testing._activate_testing() @@ -117,13 +119,13 @@ def test_gettext_lazy_proxy(): orig = u"Password" set_thread_locale("es") - p1 = unicode(proxy) + p1 = six.text_type(proxy) p1_should = pass_to_ugettext(orig) assert p1_should != orig, "Test useless, string not translated" assert p1 == p1_should set_thread_locale("sv") - p2 = unicode(proxy) + p2 = six.text_type(proxy) p2_should = pass_to_ugettext(orig) assert p2_should != orig, "Test broken, string not translated" assert p2 == p2_should diff --git a/mediagoblin/tools/url.py b/mediagoblin/tools/url.py index 657c0373..4d97247a 100644 --- a/mediagoblin/tools/url.py +++ b/mediagoblin/tools/url.py @@ -17,6 +17,8 @@ import re from unidecode import unidecode +import six + _punct_re = re.compile(r'[\t !"#:$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') @@ -27,4 +29,4 @@ def slugify(text, delim=u'-'): result = [] for word in _punct_re.split(text.lower()): result.extend(unidecode(word).split()) - return unicode(delim.join(result)) + return six.text_type(delim.join(result)) diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 78751a28..1f0b9dcd 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -18,6 +18,8 @@ import logging import datetime import json +import six + from mediagoblin import messages, mg_globals from mediagoblin.db.models import (MediaEntry, MediaTag, Collection, CollectionItem, User) @@ -178,7 +180,7 @@ def media_post_comment(request, media): comment = request.db.MediaComment() comment.media_entry = media.id comment.author = request.user.id - comment.content = unicode(request.form['comment_content']) + comment.content = six.text_type(request.form['comment_content']) # Show error message if commenting is disabled. if not mg_globals.app_config['allow_comments']: @@ -212,7 +214,7 @@ def media_preview_comment(request): if not request.is_xhr: return render_404(request) - comment = unicode(request.form['comment_content']) + comment = six.text_type(request.form['comment_content']) cleancomment = { "content":cleaned_markdown_conversion(comment)} return Response(json.dumps(cleancomment)) From 0b2572b9a8f8d1e22e724283cd99152c767208e8 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 2 Jun 2014 21:01:48 +0300 Subject: [PATCH 11/68] Add py2_unicode helper to mediagoblin._compat. --- mediagoblin/_compat.py | 7 +++++++ mediagoblin/tools/workbench.py | 10 ++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index a6b775fc..fd6172fd 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -14,3 +14,10 @@ else: import urlparse ugettext = mg_globals.thread_scope.translations.ugettext ungettext = mg_globals.thread_scope.translations.ungettext + + +def py2_unicode(klass): + if not PY3: + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass diff --git a/mediagoblin/tools/workbench.py b/mediagoblin/tools/workbench.py index 0bd4096b..f1ad6414 100644 --- a/mediagoblin/tools/workbench.py +++ b/mediagoblin/tools/workbench.py @@ -18,10 +18,15 @@ import os import shutil import tempfile +import six + +from mediagoblin._compat import py2_unicode # Actual workbench stuff # ---------------------- + +@py2_unicode class Workbench(object): """ Represent the directory for the workbench @@ -36,11 +41,8 @@ class Workbench(object): """ self.dir = dir - def __unicode__(self): - return unicode(self.dir) - def __str__(self): - return str(self.dir) + return six.text_type(self.dir) def __repr__(self): try: From 5a239cb7b8b76ae1aec8df08505b3f146f0867df Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 2 Jun 2014 21:03:56 +0300 Subject: [PATCH 12/68] Use six.moves.zip instead of itertools.izip. --- mediagoblin/tools/pagination.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tools/pagination.py b/mediagoblin/tools/pagination.py index 855878e0..a525caf7 100644 --- a/mediagoblin/tools/pagination.py +++ b/mediagoblin/tools/pagination.py @@ -17,9 +17,11 @@ import urllib import copy from math import ceil, floor -from itertools import izip, count +from itertools import count from werkzeug.datastructures import MultiDict +from six.moves import zip + PAGINATION_DEFAULT_PER_PAGE = 30 @@ -52,7 +54,7 @@ class Pagination(object): if jump_to_id: cursor = copy.copy(self.cursor) - for (doc, increment) in izip(cursor, count(0)): + for (doc, increment) in list(zip(cursor, count(0))): if doc.id == jump_to_id: self.page = 1 + int(floor(increment / self.per_page)) From f9a7201c32a42f31ac7c0c8568e80c8fbf5aec32 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 2 Jun 2014 22:25:27 +0300 Subject: [PATCH 13/68] Use mediagoblin._compat.{ugettext, ungettext} on Python 3. --- mediagoblin/_compat.py | 1 + mediagoblin/tools/translate.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index fd6172fd..c38e0727 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -7,6 +7,7 @@ from mediagoblin import mg_globals if PY3: from email.mime.text import MIMEText from urllib import parse as urlparse + # TODO(berker): Rename to gettext and ungettext instead? ugettext = mg_globals.thread_scope.translations.gettext ungettext = mg_globals.thread_scope.translations.ngettext else: diff --git a/mediagoblin/tools/translate.py b/mediagoblin/tools/translate.py index 257bd791..e6df612d 100644 --- a/mediagoblin/tools/translate.py +++ b/mediagoblin/tools/translate.py @@ -22,6 +22,7 @@ from babel import localedata from babel.support import LazyProxy from mediagoblin import mg_globals +from mediagoblin._compat import ugettext, ungettext ################### # Translation tools @@ -146,8 +147,7 @@ def pass_to_ugettext(*args, **kwargs): The reason we can't have a global ugettext method is because mg_globals gets swapped out by the application per-request. """ - return mg_globals.thread_scope.translations.ugettext( - *args, **kwargs) + return ugettext(*args, **kwargs) def pass_to_ungettext(*args, **kwargs): """ @@ -156,8 +156,7 @@ def pass_to_ungettext(*args, **kwargs): The reason we can't have a global ugettext method is because mg_globals gets swapped out by the application per-request. """ - return mg_globals.thread_scope.translations.ungettext( - *args, **kwargs) + return ungettext(*args, **kwargs) def lazy_pass_to_ugettext(*args, **kwargs): From dce76c3ee7153e8340cb3d64e8ee4df23a545bc4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Tue, 3 Jun 2014 01:41:44 +0300 Subject: [PATCH 14/68] Sync py2_unicode decorator with Django. --- mediagoblin/_compat.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index c38e0727..ab5dbcf6 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -1,5 +1,3 @@ -import sys - from six import PY3, iteritems from mediagoblin import mg_globals @@ -17,8 +15,13 @@ else: ungettext = mg_globals.thread_scope.translations.ungettext +# taken from https://github.com/django/django/blob/master/django/utils/encoding.py def py2_unicode(klass): if not PY3: + if '__str__' not in klass.__dict__: + raise ValueError("@py2_unicode cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) klass.__unicode__ = klass.__str__ klass.__str__ = lambda self: self.__unicode__().encode('utf-8') return klass From 120fa4ae95329007c529b310f4e950146d44159f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 7 Jun 2014 13:49:39 +0300 Subject: [PATCH 15/68] Change urllib and urllib import with six.moves.urllib. --- mediagoblin/processing/task.py | 12 ++++++------ mediagoblin/tools/processing.py | 11 +++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 7f683485..1a21c6d2 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -15,8 +15,8 @@ # along with this program. If not, see . import logging -import urllib -import urllib2 + +from six.moves.urllib import request, parse import celery from celery.registry import tasks @@ -42,15 +42,15 @@ def handle_push_urls(feed_url): hubparameters = { 'hub.mode': 'publish', 'hub.url': feed_url} - hubdata = urllib.urlencode(hubparameters) + hubdata = parse.urlencode(hubparameters) hubheaders = { "Content-type": "application/x-www-form-urlencoded", "Connection": "close"} for huburl in mgg.app_config["push_urls"]: - hubrequest = urllib2.Request(huburl, hubdata, hubheaders) + hubrequest = request.Request(huburl, hubdata, hubheaders) try: - hubresponse = urllib2.urlopen(hubrequest) - except (urllib2.HTTPError, urllib2.URLError) as exc: + hubresponse = request.urlopen(hubrequest) + except (request.HTTPError, request.URLError) as exc: # We retry by default 3 times before failing _log.info("PuSH url %r gave error %r", huburl, exc) try: diff --git a/mediagoblin/tools/processing.py b/mediagoblin/tools/processing.py index 2abe6452..39a329c5 100644 --- a/mediagoblin/tools/processing.py +++ b/mediagoblin/tools/processing.py @@ -18,8 +18,7 @@ import logging import json import traceback -from urllib2 import urlopen, Request, HTTPError -from urllib import urlencode +from six.moves.urllib import request, parse _log = logging.getLogger(__name__) @@ -37,10 +36,10 @@ def create_post_request(url, data, **kw): data_parser: The parser function that is used to parse the `data` argument ''' - data_parser = kw.get('data_parser', urlencode) + data_parser = kw.get('data_parser', parse.urlencode) headers = kw.get('headers', {}) - return Request(url, data_parser(data), headers=headers) + return request.Request(url, data_parser(data), headers=headers) def json_processing_callback(entry): @@ -76,11 +75,11 @@ def json_processing_callback(entry): data_parser=json.dumps) try: - urlopen(request) + request.urlopen(request) _log.debug('Processing callback for {0} sent'.format(entry)) return True - except HTTPError: + except request.HTTPError: _log.error('Failed to send callback: {0}'.format( traceback.format_exc())) From d9aced73f16b37eb24b24778f8f448769f1a7665 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 7 Jun 2014 13:51:42 +0300 Subject: [PATCH 16/68] The file() builtin has been removed in Python 3. Use open() instead. --- mediagoblin/gmg_commands/addmedia.py | 2 +- mediagoblin/media_types/ascii/processing.py | 6 +++--- mediagoblin/media_types/image/processing.py | 2 +- mediagoblin/storage/__init__.py | 4 ++-- mediagoblin/storage/cloudfiles.py | 4 ++-- mediagoblin/tests/test_pluginapi.py | 6 +++--- mediagoblin/tests/test_storage.py | 12 ++++++------ mediagoblin/tools/exif.py | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/mediagoblin/gmg_commands/addmedia.py b/mediagoblin/gmg_commands/addmedia.py index 34add88b..b741b96f 100644 --- a/mediagoblin/gmg_commands/addmedia.py +++ b/mediagoblin/gmg_commands/addmedia.py @@ -95,7 +95,7 @@ def addmedia(args): submit_media( mg_app=app, user=user, - submitted_file=file(abs_filename, 'r'), filename=filename, + submitted_file=open(abs_filename, 'r'), filename=filename, title=maybe_unicodeify(args.title), description=maybe_unicodeify(args.description), license=maybe_unicodeify(args.license), diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 6f388573..712b0692 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -95,7 +95,7 @@ class CommonAsciiProcessor(MediaProcessor): orig_file.seek(0) def store_unicode_file(self): - with file(self.process_filename, 'rb') as orig_file: + with open(self.process_filename, 'rb') as orig_file: self._detect_charset(orig_file) unicode_filepath = create_pub_filepath(self.entry, 'ascii-portable.txt') @@ -114,7 +114,7 @@ class CommonAsciiProcessor(MediaProcessor): self.entry.media_files['unicode'] = unicode_filepath def generate_thumb(self, font=None, thumb_size=None): - with file(self.process_filename, 'rb') as orig_file: + with open(self.process_filename, 'rb') as orig_file: # If no font kwarg, check config if not font: font = self.ascii_config.get('thumbnail_font', None) @@ -143,7 +143,7 @@ class CommonAsciiProcessor(MediaProcessor): thumb = converter._create_image( orig_file.read()) - with file(tmp_thumb, 'w') as thumb_file: + with open(tmp_thumb, 'w') as thumb_file: thumb.thumbnail( thumb_size, Image.ANTIALIAS) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index ae9ece24..dc6a275e 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -76,7 +76,7 @@ def resize_image(entry, resized, keyname, target_name, new_size, # Copy the new file to the conversion subdir, then remotely. tmp_resized_filename = os.path.join(workdir, target_name) - with file(tmp_resized_filename, 'w') as resized_file: + with open(tmp_resized_filename, 'wb') as resized_file: resized.save(resized_file, quality=quality) store_public(entry, keyname, tmp_resized_filename, target_name) diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 8462e08e..14f13bd3 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -178,7 +178,7 @@ class StorageInterface(object): shutil.copy(self.get_local_path(filepath), dest_path) else: with self.get_file(filepath, 'rb') as source_file: - with file(dest_path, 'wb') as dest_file: + with open(dest_path, 'wb') as dest_file: # Copy from remote storage in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) @@ -191,7 +191,7 @@ class StorageInterface(object): your storage system. """ with self.get_file(filepath, 'wb') as dest_file: - with file(filename, 'rb') as source_file: + with open(filename, 'rb') as source_file: # Copy to storage system in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py index 47c81ad6..532e5bac 100644 --- a/mediagoblin/storage/cloudfiles.py +++ b/mediagoblin/storage/cloudfiles.py @@ -143,7 +143,7 @@ class CloudFilesStorage(StorageInterface): """ # Override this method, using the "stream" iterator for efficient streaming with self.get_file(filepath, 'rb') as source_file: - with file(dest_path, 'wb') as dest_file: + with open(dest_path, 'wb') as dest_file: for data in source_file: dest_file.write(data) @@ -164,7 +164,7 @@ class CloudFilesStorage(StorageInterface): # TODO: Fixing write() still seems worthwhile though. _log.debug('Sending {0} to cloudfiles...'.format(filepath)) with self.get_file(filepath, 'wb') as dest_file: - with file(filename, 'rb') as source_file: + with open(filename, 'rb') as source_file: # Copy to storage system in 4096 byte chunks dest_file.send(source_file) diff --git a/mediagoblin/tests/test_pluginapi.py b/mediagoblin/tests/test_pluginapi.py index eae0ce15..5a3d41e6 100644 --- a/mediagoblin/tests/test_pluginapi.py +++ b/mediagoblin/tests/test_pluginapi.py @@ -421,7 +421,7 @@ def test_plugin_assetlink(static_plugin_app): junk_file_path = os.path.join( linked_assets_dir.rstrip(os.path.sep), 'junk.txt') - with file(junk_file_path, 'w') as junk_file: + with open(junk_file_path, 'w') as junk_file: junk_file.write('barf') os.unlink(plugin_link_dir) @@ -440,14 +440,14 @@ to: # link dir exists, but is a non-symlink os.unlink(plugin_link_dir) - with file(plugin_link_dir, 'w') as clobber_file: + with open(plugin_link_dir, 'w') as clobber_file: clobber_file.write('clobbered!') result = run_assetlink().collection[0] assert result == 'Could not link "staticstuff": %s exists and is not a symlink\n' % ( plugin_link_dir) - with file(plugin_link_dir, 'r') as clobber_file: + with open(plugin_link_dir, 'r') as clobber_file: assert clobber_file.read() == 'clobbered!' diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 21947415..9b96ecbe 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -174,7 +174,7 @@ def test_basic_storage_get_file(): with this_storage.get_file(filepath, 'r') as our_file: assert our_file.read() == 'First file' assert os.path.exists(os.path.join(tmpdir, 'dir1/dir2/ourfile.txt')) - with file(os.path.join(tmpdir, 'dir1/dir2/ourfile.txt'), 'r') as our_file: + with open(os.path.join(tmpdir, 'dir1/dir2/ourfile.txt'), 'r') as our_file: assert our_file.read() == 'First file' # Write to the same path but try to get a unique file. @@ -186,13 +186,13 @@ def test_basic_storage_get_file(): with this_storage.get_file(new_filepath, 'r') as our_file: assert our_file.read() == 'Second file' assert os.path.exists(os.path.join(tmpdir, *new_filepath)) - with file(os.path.join(tmpdir, *new_filepath), 'r') as our_file: + with open(os.path.join(tmpdir, *new_filepath), 'r') as our_file: assert our_file.read() == 'Second file' # Read from an existing file manually_written_file = os.makedirs( os.path.join(tmpdir, 'testydir')) - with file(os.path.join(tmpdir, 'testydir/testyfile.txt'), 'w') as testyfile: + with open(os.path.join(tmpdir, 'testydir/testyfile.txt'), 'w') as testyfile: testyfile.write('testy file! so testy.') with this_storage.get_file(['testydir', 'testyfile.txt']) as testyfile: @@ -288,7 +288,7 @@ def test_basic_storage_copy_locally(): this_storage.copy_locally(filepath, new_file_dest) this_storage.delete_file(filepath) - assert file(new_file_dest).read() == 'Testing this file' + assert open(new_file_dest).read() == 'Testing this file' os.remove(new_file_dest) os.rmdir(dest_tmpdir) @@ -297,7 +297,7 @@ def test_basic_storage_copy_locally(): def _test_copy_local_to_storage_works(tmpdir, this_storage): local_filename = tempfile.mktemp() - with file(local_filename, 'w') as tmpfile: + with open(local_filename, 'w') as tmpfile: tmpfile.write('haha') this_storage.copy_local_to_storage( @@ -305,7 +305,7 @@ def _test_copy_local_to_storage_works(tmpdir, this_storage): os.remove(local_filename) - assert file( + assert open( os.path.join(tmpdir, 'dir1/dir2/copiedto.txt'), 'r').read() == 'haha' diff --git a/mediagoblin/tools/exif.py b/mediagoblin/tools/exif.py index 736bae7d..691d3a2b 100644 --- a/mediagoblin/tools/exif.py +++ b/mediagoblin/tools/exif.py @@ -77,7 +77,7 @@ def extract_exif(filename): Returns EXIF tags found in file at ``filename`` """ try: - with file(filename) as image: + with open(filename, 'rb') as image: return process_file(image, details=False) except IOError: raise BadMediaFail(_('Could not read the image file.')) From f1d9a62a4b6c257ea22a6e33a93a4d21f249e2d1 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 7 Jun 2014 14:00:27 +0300 Subject: [PATCH 17/68] Update a comment. --- mediagoblin/_compat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index ab5dbcf6..65fb5140 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -15,7 +15,8 @@ else: ungettext = mg_globals.thread_scope.translations.ungettext -# taken from https://github.com/django/django/blob/master/django/utils/encoding.py +# taken from +# https://github.com/django/django/blob/master/django/utils/encoding.py def py2_unicode(klass): if not PY3: if '__str__' not in klass.__dict__: From 03d4be0e398525fa086a1ebb5c6e7a282f039109 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 8 Jun 2014 08:19:04 +0300 Subject: [PATCH 18/68] Fix an usage of file(). --- mediagoblin/tests/test_workbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py index 6695618b..f3ff57ed 100644 --- a/mediagoblin/tests/test_workbench.py +++ b/mediagoblin/tests/test_workbench.py @@ -50,7 +50,7 @@ class TestWorkbench(object): # kill a workbench this_workbench = self.workbench_manager.create() tmpfile_name = this_workbench.joinpath('temp.txt') - tmpfile = file(tmpfile_name, 'w') + tmpfile = open(tmpfile_name, 'w') with tmpfile: tmpfile.write('lollerskates') From bda76c420d4746217c0e27b6fa8bcde001377641 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 8 Jun 2014 08:19:32 +0300 Subject: [PATCH 19/68] Use six.iteritems() in tools/exif.py. --- mediagoblin/tools/exif.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/tools/exif.py b/mediagoblin/tools/exif.py index 691d3a2b..5de9a772 100644 --- a/mediagoblin/tools/exif.py +++ b/mediagoblin/tools/exif.py @@ -134,7 +134,7 @@ def _ratio_to_list(ratio): def get_useful(tags): - return dict((key, tag) for (key, tag) in tags.iteritems()) + return dict((key, tag) for (key, tag) in six.iteritems(tags)) def get_gps_data(tags): From ef3badb3b40809e9c8405c815663f9a427427cff Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 8 Jun 2014 08:20:17 +0300 Subject: [PATCH 20/68] Use new-style classes. The old-style classes are deprecated in Python 3. --- mediagoblin/tests/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 9b96ecbe..5cb1672b 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -47,7 +47,7 @@ def test_clean_listy_filepath(): storage.clean_listy_filepath(['../../', 'linooks.jpg']) -class FakeStorageSystem(): +class FakeStorageSystem(object): def __init__(self, foobie, blech, **kwargs): self.foobie = foobie self.blech = blech @@ -81,7 +81,7 @@ def test_storage_system_from_config(): assert this_storage.foobie == 'eiboof' assert this_storage.blech == 'hcelb' assert six.text_type(this_storage.__class__) == \ - u'mediagoblin.tests.test_storage.FakeStorageSystem' + u"" ########################## From e6aab20dc7b757406d976a468964d917e5b68ce0 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 25 Jun 2014 21:10:37 +0300 Subject: [PATCH 21/68] Use new-style class. --- mediagoblin/media_types/stl/model_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/media_types/stl/model_loader.py b/mediagoblin/media_types/stl/model_loader.py index 88f19314..c1864613 100644 --- a/mediagoblin/media_types/stl/model_loader.py +++ b/mediagoblin/media_types/stl/model_loader.py @@ -22,7 +22,7 @@ class ThreeDeeParseError(Exception): pass -class ThreeDee(): +class ThreeDee(object): """ 3D model parser base class. Derrived classes are used for basic analysis of 3D models, and are not intended to be used for 3D From e2cb0f86fe72c0f00335d7917ad23a5dd18d6e29 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 27 Jun 2014 03:17:12 +0300 Subject: [PATCH 22/68] Remove sqlalchemy-migrate imports from the codebase. Do not touch tests yet. --- mediagoblin/db/migrations.py | 2 +- mediagoblin/db/models.py | 8 -------- mediagoblin/plugins/oauth/models.py | 4 ---- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index dd96c3f3..b3cea871 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -25,7 +25,7 @@ from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import and_ -from migrate.changeset.constraint import UniqueConstraint +from sqlalchemy.schema import UniqueConstraint from mediagoblin.db.extratypes import JSONEncoded, MutationDict diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 4b2fb632..d3d1ec4b 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -42,17 +42,9 @@ from mediagoblin.tools.common import import_component import six -# It's actually kind of annoying how sqlalchemy-migrate does this, if -# I understand it right, but whatever. Anyway, don't remove this :P -# -# We could do migration calls more manually instead of relying on -# this import-based meddling... -from migrate import changeset - _log = logging.getLogger(__name__) - class User(Base, UserMixin): """ TODO: We should consider moving some rarely used fields diff --git a/mediagoblin/plugins/oauth/models.py b/mediagoblin/plugins/oauth/models.py index 439424d3..3fe562a2 100644 --- a/mediagoblin/plugins/oauth/models.py +++ b/mediagoblin/plugins/oauth/models.py @@ -26,10 +26,6 @@ from mediagoblin.db.models import User from mediagoblin.plugins.oauth.tools import generate_identifier, \ generate_secret, generate_token, generate_code, generate_refresh_token -# Don't remove this, I *think* it applies sqlalchemy-migrate functionality onto -# the models. -from migrate import changeset - class OAuthClient(Base): __tablename__ = 'oauth__client' From f9d93c0e9a8b5c1958db8771afa84519d63f5b98 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 29 Jun 2014 04:13:23 +0300 Subject: [PATCH 23/68] Fix print statements. --- mediagoblin/gmg_commands/deletemedia.py | 8 +++-- mediagoblin/gmg_commands/reprocess.py | 43 +++++++++++++------------ mediagoblin/gmg_commands/users.py | 14 ++++---- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/mediagoblin/gmg_commands/deletemedia.py b/mediagoblin/gmg_commands/deletemedia.py index d08e76cc..d8926a6b 100644 --- a/mediagoblin/gmg_commands/deletemedia.py +++ b/mediagoblin/gmg_commands/deletemedia.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function + from mediagoblin.gmg_commands import util as commands_util @@ -32,7 +34,7 @@ def deletemedia(args): for media in medias: found_medias.add(media.id) media.delete() - print 'Media ID %d has been deleted.' % media.id + print('Media ID %d has been deleted.' % media.id) for media in media_ids - found_medias: - print 'Can\'t find a media with ID %d.' % media - print 'Done.' + print('Can\'t find a media with ID %d.' % media) + print('Done.') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index e2f19ea3..85cae6df 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + +from __future__ import print_function + import argparse import os @@ -143,7 +146,7 @@ def available(args): manager = get_processing_manager_for_type(media_type) except ProcessingManagerDoesNotExist: entry = MediaEntry.query.filter_by(id=args.id_or_type).first() - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) if args.state: processors = manager.list_all_processors_by_state(args.state) @@ -152,25 +155,25 @@ def available(args): else: processors = manager.list_eligible_processors(media_entry) - print "Available processors:" - print "=====================" - print "" + print("Available processors:") + print("=====================") + print("") if args.action_help: for processor in processors: - print processor.name - print "-" * len(processor.name) + print(processor.name) + print("-" * len(processor.name)) parser = processor.generate_parser() parser.print_help() - print "" + print("") else: for processor in processors: if processor.description: - print " - %s: %s" % (processor.name, processor.description) + print(" - %s: %s" % (processor.name, processor.description)) else: - print " - %s" % processor.name + print(" - %s" % processor.name) def run(args, media_id=None): @@ -185,12 +188,12 @@ def run(args, media_id=None): processor_class = manager.get_processor( args.reprocess_command, media_entry) except ProcessorDoesNotExist: - print 'No such processor "%s" for media with id "%s"' % ( - args.reprocess_command, media_entry.id) + print('No such processor "%s" for media with id "%s"' % ( + args.reprocess_command, media_entry.id)) return except ProcessorNotEligible: - print 'Processor "%s" exists but media "%s" is not eligible' % ( - args.reprocess_command, media_entry.id) + print('Processor "%s" exists but media "%s" is not eligible' % ( + args.reprocess_command, media_entry.id)) return reprocess_parser = processor_class.generate_parser() @@ -203,7 +206,7 @@ def run(args, media_id=None): except ProcessingManagerDoesNotExist: entry = MediaEntry.query.filter_by(id=media_id).first() - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def bulk_run(args): @@ -233,12 +236,12 @@ def thumbs(args): processor_class = manager.get_processor( 'resize', media_entry) except ProcessorDoesNotExist: - print 'No such processor "%s" for media with id "%s"' % ( - 'resize', media_entry.id) + print('No such processor "%s" for media with id "%s"' % ( + 'resize', media_entry.id)) return except ProcessorNotEligible: - print 'Processor "%s" exists but media "%s" is not eligible' % ( - 'resize', media_entry.id) + print('Processor "%s" exists but media "%s" is not eligible' % ( + 'resize', media_entry.id)) return reprocess_parser = processor_class.generate_parser() @@ -260,7 +263,7 @@ def thumbs(args): reprocess_info=reprocess_request) except ProcessingManagerDoesNotExist: - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def initial(args): @@ -276,7 +279,7 @@ def initial(args): media_entry, reprocess_action='initial') except ProcessingManagerDoesNotExist: - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def reprocess(args): diff --git a/mediagoblin/gmg_commands/users.py b/mediagoblin/gmg_commands/users.py index 9014da87..050c5348 100644 --- a/mediagoblin/gmg_commands/users.py +++ b/mediagoblin/gmg_commands/users.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import print_function + import six from mediagoblin.gmg_commands import util as commands_util @@ -47,7 +49,7 @@ def adduser(args): ).count() if users_with_username: - print u'Sorry, a user with that name already exists.' + print(u'Sorry, a user with that name already exists.') else: # Create the user @@ -68,7 +70,7 @@ def adduser(args): entry.all_privileges = default_privileges entry.save() - print "User created (and email marked as verified)" + print(u"User created (and email marked as verified)") def makeadmin_parser_setup(subparser): @@ -90,9 +92,9 @@ def makeadmin(args): db.Privilege.privilege_name==u'admin').one() ) user.save() - print 'The user is now Admin' + print(u'The user is now Admin') else: - print 'The user doesn\'t exist' + print(u'The user doesn\'t exist') def changepw_parser_setup(subparser): @@ -114,6 +116,6 @@ def changepw(args): if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() - print 'Password successfully changed' + print(u'Password successfully changed') else: - print 'The user doesn\'t exist' + print(u'The user doesn\'t exist') From 19baab1b034b31a53a0dbddd17d3f31594ce5afc Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 29 Jun 2014 04:23:50 +0300 Subject: [PATCH 24/68] Remove Paste#http, Paste#urlmap and Paste#static dependencies. Changes: * Paste#http -> Gunicorn * Paste#urlmap and Paste#static -> werkzeug.wsgi.SharedDataMiddleware --- lazystarter.sh | 9 +++---- mediagoblin/app.py | 4 +++ paste.ini | 66 +++++++++------------------------------------- 3 files changed, 20 insertions(+), 59 deletions(-) diff --git a/lazystarter.sh b/lazystarter.sh index d3770194..2842d5d5 100755 --- a/lazystarter.sh +++ b/lazystarter.sh @@ -20,7 +20,7 @@ selfname=$(basename "$0") local_bin="./bin" case "$selfname" in lazyserver.sh) - starter_cmd=paster + starter_cmd=gunicorn ini_prefix=paste ;; lazycelery.sh) @@ -36,9 +36,8 @@ esac if [ "$1" = "-h" ]; then echo "$0 [-h] [-c filename.ini] [ARGS_to_${starter_cmd} ...]" echo "" - echo " For example:" - echo " $0 -c fcgi.ini port_number=23371" - echo " or: $0 --server-name=fcgi --log-file=paste.log" + echo " For Gunicorn settings, see at:" + echo " http://docs.gunicorn.org/en/19.0/settings.html" echo "" echo " The configfile defaults to ${ini_prefix}_local.ini," echo " if that is readable, otherwise ${ini_prefix}.ini." @@ -71,7 +70,7 @@ set -x export CELERY_ALWAYS_EAGER=true case "$selfname" in lazyserver.sh) - $starter serve "$ini_file" "$@" --reload + $starter --paste "$ini_file" "$@" ;; lazycelery.sh) MEDIAGOBLIN_CONFIG="${ini_file}" \ diff --git a/mediagoblin/app.py b/mediagoblin/app.py index e65e6d10..d912206e 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -23,6 +23,7 @@ from mediagoblin.tools.routing import endpoint_to_controller from werkzeug.wrappers import Request from werkzeug.exceptions import HTTPException from werkzeug.routing import RequestRedirect +from werkzeug.wsgi import SharedDataMiddleware from mediagoblin import meddleware, __version__ from mediagoblin.db.util import check_db_up_to_date @@ -277,8 +278,11 @@ def paste_app_factory(global_config, **app_config): if not mediagoblin_config: raise IOError("Usable mediagoblin config not found.") + del app_config['config'] mgoblin_app = MediaGoblinApp(mediagoblin_config) + mgoblin_app.call_backend = SharedDataMiddleware(mgoblin_app.call_backend, + exports=app_config) mgoblin_app = hook_transform('wrap_wsgi', mgoblin_app) return mgoblin_app diff --git a/paste.ini b/paste.ini index 3c7eb177..b859251e 100644 --- a/paste.ini +++ b/paste.ini @@ -6,19 +6,16 @@ debug = false [pipeline:main] -pipeline = errors routing - -[composite:routing] -use = egg:Paste#urlmap -/ = mediagoblin -/mgoblin_media/ = publicstore_serve -/mgoblin_static/ = mediagoblin_static -/theme_static/ = theme_static -/plugin_static/ = plugin_static +pipeline = errors mediagoblin [app:mediagoblin] use = egg:mediagoblin#app config = %(here)s/mediagoblin_local.ini %(here)s/mediagoblin.ini +# static paths +/mgoblin_media = %(here)s/user_dev/media/public +/mgoblin_static = %(here)s/mediagoblin/static +/theme_static = %(here)s/user_dev/theme_static +/plugin_static = %(here)s/user_dev/plugin_static [loggers] keys = root @@ -42,26 +39,6 @@ formatter = generic [formatter_generic] format = %(asctime)s %(levelname)-7.7s [%(name)s] %(message)s -[app:publicstore_serve] -use = egg:Paste#static -document_root = %(here)s/user_dev/media/public/ -cache_max_age = 604800 - -[app:mediagoblin_static] -use = egg:Paste#static -document_root = %(here)s/mediagoblin/static/ -cache_max_age = 86400 - -[app:theme_static] -use = egg:Paste#static -document_root = %(here)s/user_dev/theme_static/ -cache_max_age = 86400 - -[app:plugin_static] -use = egg:Paste#static -document_root = %(here)s/user_dev/plugin_static/ -cache_max_age = 86400 - [filter:errors] use = egg:mediagoblin#errors debug = false @@ -74,30 +51,11 @@ debug = false # The server that is run by default. # By default, should only be accessable locally [server:main] -use = egg:Paste#http +use = egg:gunicorn host = 127.0.0.1 port = 6543 - -####################### -# Helper server configs -# --------------------- -# If you are configuring the paste config manually, you can remove -# these. - -# Use this if you want to run on port 6543 and have MediaGoblin be -# viewable externally -[server:broadcast] -use = egg:Paste#http -host = 0.0.0.0 -port = 6543 - -# Use this if you want to connect via fastcgi -[server:fcgi] -use = egg:flup#fcgi_fork -host = %(fcgi_host)s -port = %(fcgi_port)s - -[server:http] -use = egg:Paste#http -host = %(http_host)s -port = %(http_port)s +# Gunicorn settings. See http://docs.gunicorn.org/en/19.0/settings.html +# for more information about configuring Gunicorn +proc_name = gmg +reload = true +accesslog = - From 9459fa3cede5b1a1a061cc5a15a25b2715d0d47d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 14 Jul 2014 05:49:38 +0300 Subject: [PATCH 25/68] Fix tests on Python 3. --- mediagoblin/tests/test_api.py | 4 +-- mediagoblin/tests/test_edit.py | 2 +- mediagoblin/tests/test_exif.py | 12 +++++-- mediagoblin/tests/test_http_callback.py | 2 +- mediagoblin/tests/test_ldap.py | 5 ++- mediagoblin/tests/test_modelmethods.py | 9 ++++-- mediagoblin/tests/test_oauth1.py | 13 ++++---- mediagoblin/tests/test_oauth2.py | 2 +- mediagoblin/tests/test_openid.py | 7 +++-- mediagoblin/tests/test_persona.py | 5 ++- mediagoblin/tests/test_pluginapi.py | 6 ++-- mediagoblin/tests/test_privileges.py | 42 ++++++++++++++++--------- mediagoblin/tests/test_submission.py | 10 +++--- mediagoblin/tools/exif.py | 7 +++-- 14 files changed, 81 insertions(+), 45 deletions(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 4e0cbd8f..e19313e4 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -57,7 +57,7 @@ class TestAPI(object): url = kwargs.pop('url', '/api/submit') do_follow = kwargs.pop('do_follow', False) - if not 'headers' in kwargs.keys(): + if 'headers' not in kwargs.keys(): kwargs['headers'] = self.http_auth_headers() response = test_app.post(url, data, **kwargs) @@ -78,7 +78,7 @@ class TestAPI(object): headers=self.http_auth_headers()) assert response.body == \ - '{"username": "joapi", "email": "joapi@example.com"}' + b'{"email": "joapi@example.com", "username": "joapi"}' def test_2_test_submission(self, test_app): self.login(test_app) diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index 4f44e0b9..b60d4c74 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse +import six.moves.urllib.parse as urlparse from mediagoblin import mg_globals from mediagoblin.db.models import User diff --git a/mediagoblin/tests/test_exif.py b/mediagoblin/tests/test_exif.py index af301818..861b768a 100644 --- a/mediagoblin/tests/test_exif.py +++ b/mediagoblin/tests/test_exif.py @@ -20,6 +20,8 @@ try: except ImportError: import Image +from collections import OrderedDict + from mediagoblin.tools.exif import exif_fix_image_orientation, \ extract_exif, clean_exif, get_gps_data, get_useful from .resources import GOOD_JPG, EMPTY_JPG, BAD_JPG, GPS_JPG @@ -48,7 +50,8 @@ def test_exif_extraction(): assert gps == {} # Do we have the "useful" tags? - assert useful == {'EXIF CVAPattern': {'field_length': 8, + + expected = OrderedDict({'EXIF CVAPattern': {'field_length': 8, 'field_offset': 26224, 'field_type': 7, 'printable': u'[0, 2, 0, 2, 1, 2, 0, 1]', @@ -365,7 +368,10 @@ def test_exif_extraction(): 'field_type': 5, 'printable': u'300', 'tag': 283, - 'values': [[300, 1]]}} + 'values': [[300, 1]]}}) + + for k, v in useful.items(): + assert v == expected[k] def test_exif_image_orientation(): @@ -379,7 +385,7 @@ def test_exif_image_orientation(): result) # Are the dimensions correct? - assert image.size == (428, 640) + assert image.size in ((428, 640), (640, 428)) # If this pixel looks right, the rest of the image probably will too. assert_in(image.getdata()[10000], diff --git a/mediagoblin/tests/test_http_callback.py b/mediagoblin/tests/test_http_callback.py index d0a8c823..11f02c97 100644 --- a/mediagoblin/tests/test_http_callback.py +++ b/mediagoblin/tests/test_http_callback.py @@ -19,7 +19,7 @@ import json import pytest import six -from urlparse import urlparse, parse_qs +from six.moves.urllib.parse import parse_qs, urlparse from mediagoblin import mg_globals from mediagoblin.tools import processing diff --git a/mediagoblin/tests/test_ldap.py b/mediagoblin/tests/test_ldap.py index e69a3203..9164da78 100644 --- a/mediagoblin/tests/test_ldap.py +++ b/mediagoblin/tests/test_ldap.py @@ -16,8 +16,11 @@ import pkg_resources import pytest -import mock import six +try: + import mock +except ImportError: + import unittest.mock as mock import six.moves.urllib.parse as urlparse diff --git a/mediagoblin/tests/test_modelmethods.py b/mediagoblin/tests/test_modelmethods.py index ca436c76..d2d6bdcf 100644 --- a/mediagoblin/tests/test_modelmethods.py +++ b/mediagoblin/tests/test_modelmethods.py @@ -17,13 +17,18 @@ # Maybe not every model needs a test, but some models have special # methods, and so it makes sense to test them here. +from __future__ import print_function + from mediagoblin.db.base import Session from mediagoblin.db.models import MediaEntry, User, Privilege from mediagoblin.tests import MGClientTestCase from mediagoblin.tests.tools import fixture_add_user -import mock +try: + import mock +except ImportError: + import unittest.mock as mock import pytest @@ -205,7 +210,7 @@ def test_media_data_init(test_app): obj_in_session = 0 for obj in Session(): obj_in_session += 1 - print repr(obj) + print(repr(obj)) assert obj_in_session == 0 diff --git a/mediagoblin/tests/test_oauth1.py b/mediagoblin/tests/test_oauth1.py index 073c2884..16c0f280 100644 --- a/mediagoblin/tests/test_oauth1.py +++ b/mediagoblin/tests/test_oauth1.py @@ -17,7 +17,8 @@ import cgi import pytest -from urlparse import parse_qs, urlparse + +from six.moves.urllib.parse import parse_qs, urlparse from oauthlib.oauth1 import Client @@ -52,8 +53,8 @@ class TestOAuth(object): def register_client(self, **kwargs): """ Regiters a client with the API """ - - kwargs["type"] = "client_associate" + + kwargs["type"] = "client_associate" kwargs["application_type"] = kwargs.get("application_type", "native") return self.test_app.post("/api/client/register", kwargs) @@ -63,7 +64,7 @@ class TestOAuth(object): client_info = response.json client = self.db.Client.query.filter_by(id=client_info["client_id"]).first() - + assert response.status_int == 200 assert client is not None @@ -81,7 +82,7 @@ class TestOAuth(object): client_info = response.json client = self.db.Client.query.filter_by(id=client_info["client_id"]).first() - + assert client is not None assert client.secret == client_info["client_secret"] assert client.application_type == query["application_type"] @@ -163,4 +164,4 @@ class TestOAuth(object): assert request_token.client == client.id assert request_token.used == False assert request_token.callback == request_query["oauth_callback"] - + diff --git a/mediagoblin/tests/test_oauth2.py b/mediagoblin/tests/test_oauth2.py index 6bdb729e..014808a6 100644 --- a/mediagoblin/tests/test_oauth2.py +++ b/mediagoblin/tests/test_oauth2.py @@ -20,7 +20,7 @@ import logging import pytest import six -from urlparse import parse_qs, urlparse +from six.moves.urllib.parse import parse_qs, urlparse from mediagoblin import mg_globals from mediagoblin.tools import template, pluginapi diff --git a/mediagoblin/tests/test_openid.py b/mediagoblin/tests/test_openid.py index 7ef01052..a3ab176a 100644 --- a/mediagoblin/tests/test_openid.py +++ b/mediagoblin/tests/test_openid.py @@ -14,11 +14,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse import pkg_resources import pytest -import mock import six +import six.moves.urllib.parse as urlparse +try: + import mock +except ImportError: + import unittest.mock as mock openid_consumer = pytest.importorskip( "openid.consumer.consumer") diff --git a/mediagoblin/tests/test_persona.py b/mediagoblin/tests/test_persona.py index f2dd4001..a8466b8a 100644 --- a/mediagoblin/tests/test_persona.py +++ b/mediagoblin/tests/test_persona.py @@ -16,8 +16,11 @@ import pkg_resources import pytest -import mock import six +try: + import mock +except ImportError: + import unittest.mock as mock import six.moves.urllib.parse as urlparse diff --git a/mediagoblin/tests/test_pluginapi.py b/mediagoblin/tests/test_pluginapi.py index 5a3d41e6..b3f37e2a 100644 --- a/mediagoblin/tests/test_pluginapi.py +++ b/mediagoblin/tests/test_pluginapi.py @@ -224,7 +224,7 @@ def test_hook_handle(): assert pluginapi.hook_handle( "nothing_handling", call_log, unhandled_okay=True) is None assert call_log == [] - + # Multiple provided, go with the first! call_log = [] assert pluginapi.hook_handle( @@ -348,7 +348,7 @@ def test_modify_context(context_modified_app): """ # Specific thing passed into a page result = context_modified_app.get("/modify_context/specific/") - assert result.body.strip() == """Specific page! + assert result.body.strip() == b"""Specific page! specific thing: in yer specificpage global thing: globally appended! @@ -357,7 +357,7 @@ doubleme: happyhappy""" # General test, should have global context variable only result = context_modified_app.get("/modify_context/") - assert result.body.strip() == """General page! + assert result.body.strip() == b"""General page! global thing: globally appended! lol: cats diff --git a/mediagoblin/tests/test_privileges.py b/mediagoblin/tests/test_privileges.py index 05829b34..8ea3d754 100644 --- a/mediagoblin/tests/test_privileges.py +++ b/mediagoblin/tests/test_privileges.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six import pytest from datetime import date, timedelta from webtest import AppError @@ -79,7 +80,7 @@ class TestPrivilegeFunctionality: response = self.test_app.get('/') assert response.status == "200 OK" - assert "You are Banned" in response.body + assert b"You are Banned" in response.body # Then test what happens when that ban has an expiration date which # hasn't happened yet #---------------------------------------------------------------------- @@ -92,7 +93,7 @@ class TestPrivilegeFunctionality: response = self.test_app.get('/') assert response.status == "200 OK" - assert "You are Banned" in response.body + assert b"You are Banned" in response.body # Then test what happens when that ban has an expiration date which # has already happened @@ -107,7 +108,7 @@ class TestPrivilegeFunctionality: response = self.test_app.get('/') assert response.status == "302 FOUND" - assert not "You are Banned" in response.body + assert not b"You are Banned" in response.body def testVariousPrivileges(self): # The various actions that require privileges (ex. reporting, @@ -127,14 +128,16 @@ class TestPrivilegeFunctionality: #---------------------------------------------------------------------- with pytest.raises(AppError) as excinfo: response = self.test_app.get('/submit/') - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.do_post({'upload_files':[('file',GOOD_JPG)], 'title':u'Normal Upload 1'}, url='/submit/') - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo # Test that a user cannot comment without the commenter privilege #---------------------------------------------------------------------- @@ -149,50 +152,58 @@ class TestPrivilegeFunctionality: media_uri_slug = '/u/{0}/m/{1}/'.format(self.admin_user.username, media_entry.slug) response = self.test_app.get(media_uri_slug) - assert not "Add a comment" in response.body + assert not b"Add a comment" in response.body self.query_for_users() with pytest.raises(AppError) as excinfo: response = self.test_app.post( media_uri_id + 'comment/add/', {'comment_content': u'Test comment #42'}) - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo # Test that a user cannot report without the reporter privilege #---------------------------------------------------------------------- with pytest.raises(AppError) as excinfo: response = self.test_app.get(media_uri_slug+"report/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.do_post( {'report_reason':u'Testing Reports #1', 'reporter_id':u'3'}, url=(media_uri_slug+"report/")) - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo # Test that a user cannot access the moderation pages w/o moderator # or admin privileges #---------------------------------------------------------------------- with pytest.raises(AppError) as excinfo: response = self.test_app.get("/mod/users/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.test_app.get("/mod/reports/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.test_app.get("/mod/media/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.test_app.get("/mod/users/1/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo with pytest.raises(AppError) as excinfo: response = self.test_app.get("/mod/reports/1/") - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo self.query_for_users() @@ -202,4 +213,5 @@ class TestPrivilegeFunctionality: 'targeted_user':self.admin_user.id}, url='/mod/reports/1/') self.query_for_users() - assert 'Bad response: 403 FORBIDDEN' in str(excinfo) + excinfo = str(excinfo) if six.PY2 else str(excinfo).encode('ascii') + assert b'Bad response: 403 FORBIDDEN' in excinfo diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index 6143c0ac..1c2c280e 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -14,13 +14,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import sys -reload(sys) -sys.setdefaultencoding('utf-8') +import six + +if six.PY2: # this hack only work in Python 2 + import sys + reload(sys) + sys.setdefaultencoding('utf-8') import os import pytest -import six import six.moves.urllib.parse as urlparse diff --git a/mediagoblin/tools/exif.py b/mediagoblin/tools/exif.py index 5de9a772..ec83f43c 100644 --- a/mediagoblin/tools/exif.py +++ b/mediagoblin/tools/exif.py @@ -112,7 +112,7 @@ def _ifd_tag_to_dict(tag): 'field_length': tag.field_length, 'values': None} - if isinstance(tag.printable, str): + if isinstance(tag.printable, six.binary_type): # Force it to be decoded as UTF-8 so that it'll fit into the DB data['printable'] = tag.printable.decode('utf8', 'replace') @@ -120,7 +120,7 @@ def _ifd_tag_to_dict(tag): data['values'] = [_ratio_to_list(val) if isinstance(val, Ratio) else val for val in tag.values] else: - if isinstance(tag.values, str): + if isinstance(tag.values, six.binary_type): # Force UTF-8, so that it fits into the DB data['values'] = tag.values.decode('utf8', 'replace') else: @@ -134,7 +134,8 @@ def _ratio_to_list(ratio): def get_useful(tags): - return dict((key, tag) for (key, tag) in six.iteritems(tags)) + from collections import OrderedDict + return OrderedDict((key, tag) for (key, tag) in six.iteritems(tags)) def get_gps_data(tags): From c9cbc7e8a7bca8cedb02bbcfaf17c4c17176821b Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 18 Jul 2014 11:05:11 +0300 Subject: [PATCH 26/68] Add tox.ini. --- tox.ini | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..35a99ad3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +[tox] +envlist = py27, py33 +skipsdist = True +usedevelop = True + +[testenv] +whitelist_externals = sh +commands = + python setup.py develop + sh runtests.sh +deps = + git+https://github.com/ianare/exif-py.git@develop + hg+https://bitbucket.org/berkerpeksag/paste + lxml + Pillow From 88a5739d36f5623229f359d0453988d1806e3a77 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 18 Jul 2014 12:39:13 +0300 Subject: [PATCH 27/68] Remove paste.server dependency from tests. --- mediagoblin/tests/test_paste.ini | 34 ++++++-------------------------- setup.py | 1 + tox.ini | 2 +- 3 files changed, 8 insertions(+), 29 deletions(-) diff --git a/mediagoblin/tests/test_paste.ini b/mediagoblin/tests/test_paste.ini index a9595432..8d75c3cb 100644 --- a/mediagoblin/tests/test_paste.ini +++ b/mediagoblin/tests/test_paste.ini @@ -1,40 +1,18 @@ [DEFAULT] debug = true -[composite:main] -use = egg:Paste#urlmap -/ = mediagoblin -/mgoblin_media/ = publicstore_serve -/test_static/ = mediagoblin_static -/theme_static/ = theme_static -/plugin_static/ = plugin_static - -[app:mediagoblin] +[app:main] use = egg:mediagoblin#app config = %(here)s/mediagoblin.ini - -[app:publicstore_serve] -use = egg:Paste#static -document_root = %(here)s/user_dev/media/public - -[app:mediagoblin_static] -use = egg:Paste#static -document_root = %(here)s/mediagoblin/static/ - -[app:theme_static] -use = egg:Paste#static -document_root = %(here)s/user_dev/theme_static/ -cache_max_age = 86400 - -[app:plugin_static] -use = egg:Paste#static -document_root = %(here)s/user_dev/plugin_static/ -cache_max_age = 86400 +/mgoblin_media = %(here)s/user_dev/media/public +/test_static = %(here)s/mediagoblin/static +/theme_static = %(here)s/user_dev/theme_static +/plugin_static = %(here)s/user_dev/plugin_static [celery] CELERY_ALWAYS_EAGER = true [server:main] -use = egg:Paste#http +use = egg:gunicorn host = 127.0.0.1 port = 6543 diff --git a/setup.py b/setup.py index 658cf8c3..4e4244ae 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ if PY2: py2_only_install_requires.append('mock') # mock is in the stdlib for 3.3+ install_requires = [ + 'gunicorn==19', 'setuptools', # TODO: is this necessary 'python-dateutil', 'wtforms', diff --git a/tox.ini b/tox.ini index 35a99ad3..4f123c1b 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,9 @@ usedevelop = True whitelist_externals = sh commands = python setup.py develop + gmg dbupdate sh runtests.sh deps = git+https://github.com/ianare/exif-py.git@develop - hg+https://bitbucket.org/berkerpeksag/paste lxml Pillow From 98d8b365aff5b5ff8732dbc45d45e5174795f817 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 10:35:14 +0300 Subject: [PATCH 28/68] func_name removed in Python 3. --- mediagoblin/db/migration_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index e725f565..1967dacd 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -232,7 +232,7 @@ class MigrationManager(object): for migration_number, migration_func in migrations_to_run: self.printer( u' + Running migration %s, "%s"... ' % ( - migration_number, migration_func.func_name)) + migration_number, migration_func.__name__)) migration_func(self.session) self.set_current_migration(migration_number) self.printer('done.\n') From fe6f82be237808ba86688a117f32cf2311c2118c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 12:54:59 +0300 Subject: [PATCH 29/68] Remove an usage of deprecated cgi.parse_qs(). --- mediagoblin/tests/test_oauth1.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mediagoblin/tests/test_oauth1.py b/mediagoblin/tests/test_oauth1.py index 16c0f280..f681a6b3 100644 --- a/mediagoblin/tests/test_oauth1.py +++ b/mediagoblin/tests/test_oauth1.py @@ -14,8 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import cgi - import pytest from six.moves.urllib.parse import parse_qs, urlparse @@ -147,7 +145,7 @@ class TestOAuth(object): headers["Content-Type"] = self.MIME_FORM response = self.test_app.post(endpoint, headers=headers) - response = cgi.parse_qs(response.body) + response = parse_qs(response.body.decode()) # each element is a list, reduce it to a string for key, value in response.items(): From 88ed537a9e1d85bb8f86d9a4d1289fb8d50bb073 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 13:22:42 +0300 Subject: [PATCH 30/68] Skip test_sql_migrations on Python 3 for now. --- mediagoblin/tests/test_sql_migrations.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index e86dcd80..7e0569ad 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -14,9 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import copy - import six +import pytest + +pytestmark = pytest.mark.skipif(six.PY3, reason='needs sqlalchemy.migrate') + +import copy from sqlalchemy import ( Table, Column, MetaData, Index, @@ -25,7 +28,8 @@ from sqlalchemy import ( from sqlalchemy.orm import sessionmaker, relationship from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import select, insert -from migrate import changeset +if six.PY2: + from migrate import changeset from mediagoblin.db.base import GMGTableBase from mediagoblin.db.migration_tools import MigrationManager, RegisterMigration From 173099ad2b62be700a6d0b52213412f4f3b51016 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 13:24:17 +0300 Subject: [PATCH 31/68] Update dependency list. --- setup.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 4e4244ae..f9c7f70a 100644 --- a/setup.py +++ b/setup.py @@ -47,11 +47,14 @@ if PY2: # TERRIBLE AND IS THE END OF ALL THINGS # I'd love to remove this restriction. py2_only_install_requires.append('sqlalchemy-migrate<0.8') + # Annoying. Please remove once we can! We only indirectly + # use pbr, and currently it breaks things, presumably till + # their next release. + py2_only_install_requires.append('pbr==0.5.22') py2_only_install_requires.append('mock') # mock is in the stdlib for 3.3+ install_requires = [ - 'gunicorn==19', - 'setuptools', # TODO: is this necessary + 'gunicorn==19', # TODO: Upgrade to 19.2 -- see https://github.com/benoitc/gunicorn/issues/830 'python-dateutil', 'wtforms', 'py-bcrypt', @@ -61,8 +64,7 @@ install_requires = [ 'celery>=3.0', 'kombu', 'jinja2', - 'sphinx', # TODO: is this a docs requirement? - 'Babel<1.0', # TODO: why <1.0 or 0.9.6? + 'Babel>=1.3', 'webtest<2', 'ConfigObj', 'Markdown', @@ -72,20 +74,16 @@ install_requires = [ # PLEASE change this when we can; a dependency is forcing us to set this # specific number and it is breaking setup.py develop 'six==1.5.2', - 'oauthlib==0.5.0', + 'oauthlib>=0.5.0', 'unidecode', 'ExifRead', - # Annoying. Please remove once we can! We only indirectly - # use pbr, and currently it breaks things, presumably till - # their next release. - 'pbr==0.5.22', 'PasteDeploy', # This is optional: # 'translitcodec', # For now we're expecting that users will install this from # their package managers. # 'lxml', - # 'PIL', # TODO: switch to Pillow? + # 'Pillow', ] + py2_only_install_requires with open(READMEFILE) as fobj: From cf3b5926f49a59d1547c2e4f098b9917c70168d0 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 13:27:19 +0300 Subject: [PATCH 32/68] Cleanup mediagoblin._compat and fix translation tests. --- mediagoblin/_compat.py | 12 ++---------- mediagoblin/db/open.py | 5 +++-- mediagoblin/plugins/oauth/views.py | 2 +- mediagoblin/storage/filestorage.py | 10 +++++----- mediagoblin/tools/template.py | 10 +++++++--- mediagoblin/tools/translate.py | 10 +++++++--- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index 65fb5140..5a3fac53 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -1,23 +1,15 @@ -from six import PY3, iteritems - -from mediagoblin import mg_globals +from six import PY3 if PY3: from email.mime.text import MIMEText - from urllib import parse as urlparse - # TODO(berker): Rename to gettext and ungettext instead? - ugettext = mg_globals.thread_scope.translations.gettext - ungettext = mg_globals.thread_scope.translations.ngettext else: from email.MIMEText import MIMEText - import urlparse - ugettext = mg_globals.thread_scope.translations.ugettext - ungettext = mg_globals.thread_scope.translations.ungettext # taken from # https://github.com/django/django/blob/master/django/utils/encoding.py def py2_unicode(klass): + # TODO: Add support for __repr__ if not PY3: if '__str__' not in klass.__dict__: raise ValueError("@py2_unicode cannot be applied " diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index 9cf9e578..34f0bffa 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -18,9 +18,10 @@ from sqlalchemy import create_engine, event import logging +import six + from mediagoblin.db.base import Base, Session from mediagoblin import mg_globals -from mediagoblin._compat import iteritems _log = logging.getLogger(__name__) @@ -29,7 +30,7 @@ class DatabaseMaster(object): def __init__(self, engine): self.engine = engine - for k, v in iteritems(Base._decl_class_registry): + for k, v in six.iteritems(Base._decl_class_registry): setattr(self, k, v) def commit(self): diff --git a/mediagoblin/plugins/oauth/views.py b/mediagoblin/plugins/oauth/views.py index 9e3b87c9..8ca73521 100644 --- a/mediagoblin/plugins/oauth/views.py +++ b/mediagoblin/plugins/oauth/views.py @@ -17,7 +17,7 @@ import logging -from urllib import urlencode +from six.moves.urllib.parse import urlencode import six diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 404d24c3..917c7703 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -14,16 +14,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os +import shutil + +from six.moves.urllib.parse import urlparse + from mediagoblin.storage import ( StorageInterface, clean_listy_filepath, NoWebServing) -import os -import shutil - -from mediagoblin._compat import urlparse - class BasicFileStorage(StorageInterface): """ diff --git a/mediagoblin/tools/template.py b/mediagoblin/tools/template.py index 359ddd38..b01196fd 100644 --- a/mediagoblin/tools/template.py +++ b/mediagoblin/tools/template.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six import jinja2 from jinja2.ext import Extension @@ -33,8 +34,6 @@ from mediagoblin.tools.pluginapi import get_hook_templates, hook_transform from mediagoblin.tools.timesince import timesince from mediagoblin.meddleware.csrf import render_csrf_form_token -from mediagoblin._compat import ugettext, ungettext - SETUP_JINJA_ENVS = {} @@ -67,7 +66,12 @@ def get_jinja_env(template_loader, locale): 'jinja2.ext.i18n', 'jinja2.ext.autoescape', TemplateHookExtension] + local_exts) - template_env.install_gettext_callables(ugettext, ungettext) + if six.PY2: + template_env.install_gettext_callables(mg_globals.thread_scope.translations.ugettext, + mg_globals.thread_scope.translations.ungettext) + else: + template_env.install_gettext_callables(mg_globals.thread_scope.translations.gettext, + mg_globals.thread_scope.translations.ngettext) # All templates will know how to ... # ... fetch all waiting messages and remove them from the queue diff --git a/mediagoblin/tools/translate.py b/mediagoblin/tools/translate.py index e6df612d..f8a59aee 100644 --- a/mediagoblin/tools/translate.py +++ b/mediagoblin/tools/translate.py @@ -17,12 +17,12 @@ import gettext import pkg_resources +import six from babel import localedata from babel.support import LazyProxy from mediagoblin import mg_globals -from mediagoblin._compat import ugettext, ungettext ################### # Translation tools @@ -147,7 +147,9 @@ def pass_to_ugettext(*args, **kwargs): The reason we can't have a global ugettext method is because mg_globals gets swapped out by the application per-request. """ - return ugettext(*args, **kwargs) + if six.PY2: + return mg_globals.thread_scope.translations.ugettext(*args, **kwargs) + return mg_globals.thread_scope.translations.gettext(*args, **kwargs) def pass_to_ungettext(*args, **kwargs): """ @@ -156,7 +158,9 @@ def pass_to_ungettext(*args, **kwargs): The reason we can't have a global ugettext method is because mg_globals gets swapped out by the application per-request. """ - return ungettext(*args, **kwargs) + if six.PY2: + return mg_globals.thread_scope.translations.ungettext(*args, **kwargs) + return mg_globals.thread_scope.translations.ngettext(*args, **kwargs) def lazy_pass_to_ugettext(*args, **kwargs): From 874439bd80e209129fcd56b169a734906c9f8968 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 13:29:26 +0300 Subject: [PATCH 33/68] Update .gitignore. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 6c138218..34399cad 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ # The legacy of buildout .installed.cfg + +# Virtualenv, tox +venv* +.tox/ From 15c3461b1f168b154419ded41498c9adc4ab60be Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 15:24:49 +0300 Subject: [PATCH 34/68] Disable cache support of LazyProxy. Since Babel 1.0, there's a enable_cache keyword argument of LazyProxy, but we can't pass it directly. --- mediagoblin/tools/translate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediagoblin/tools/translate.py b/mediagoblin/tools/translate.py index f8a59aee..5ea73b71 100644 --- a/mediagoblin/tools/translate.py +++ b/mediagoblin/tools/translate.py @@ -53,9 +53,9 @@ class ReallyLazyProxy(LazyProxy): """ Like LazyProxy, except that it doesn't cache the value ;) """ - @property - def value(self): - return self._func(*self._args, **self._kwargs) + def __init__(self, func, *args, **kwargs): + super(ReallyLazyProxy, self).__init__(func, *args, **kwargs) + object.__setattr__(self, '_is_cache_enabled', False) def __repr__(self): return "<%s for %s(%r, %r)>" % ( From 7ec699445364c3e0ca32f905c499e21221be27f9 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 15:25:20 +0300 Subject: [PATCH 35/68] Fix urlparse import. --- mediagoblin/storage/filestorage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 917c7703..f989539c 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -17,7 +17,7 @@ import os import shutil -from six.moves.urllib.parse import urlparse +import six.moves.urllib.parse as urlparse from mediagoblin.storage import ( StorageInterface, From 20238f54a660047b72a1c25bce14f3bd3e22f03d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Jul 2014 15:26:13 +0300 Subject: [PATCH 36/68] Use isinstance and six.text_type to check types of an object. --- mediagoblin/oauth/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mediagoblin/oauth/views.py b/mediagoblin/oauth/views.py index 14c8ab14..fd848467 100644 --- a/mediagoblin/oauth/views.py +++ b/mediagoblin/oauth/views.py @@ -16,6 +16,8 @@ import datetime +import six + from oauthlib.oauth1 import (RequestTokenEndpoint, AuthorizationEndpoint, AccessTokenEndpoint) @@ -136,7 +138,7 @@ def client_register(request): contacts = data.get("contacts", None) if contacts is not None: - if type(contacts) is not unicode: + if not isinstance(contacts, six.text_type): error = "Contacts must be a string of space-seporated email addresses." return json_response({"error": error}, status=400) @@ -152,7 +154,7 @@ def client_register(request): redirect_uris = data.get("redirect_uris", None) if redirect_uris is not None: - if type(redirect_uris) is not unicode: + if not isinstance(redirect_uris, six.text_type): error = "redirect_uris must be space-seporated URLs." return json_response({"error": error}, status=400) From 6fa978241583b8d618cd22bfb53f657a64580c03 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 4 Aug 2014 22:06:40 +0300 Subject: [PATCH 37/68] Make sort_keys True to avoid hash randomize feature in Python 3. --- mediagoblin/plugins/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/plugins/api/views.py b/mediagoblin/plugins/api/views.py index a9aaacee..02fd8107 100644 --- a/mediagoblin/plugins/api/views.py +++ b/mediagoblin/plugins/api/views.py @@ -104,7 +104,7 @@ def api_test(request): # TODO: This is the *only* thing using Response() here, should that # not simply use json_response()? - return Response(json.dumps(user_data)) + return Response(json.dumps(user_data, sort_keys=True)) def get_entries(request): From a7e1d8829f5f50813ce98b28a0b5348ac6916631 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 4 Aug 2014 22:08:23 +0300 Subject: [PATCH 38/68] mbox_message.get_payload() returns bytestring. --- mediagoblin/tests/test_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index e108e9bb..e239d628 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -54,7 +54,7 @@ I hope you like unit tests JUST AS MUCH AS I DO!""") assert message['From'] == "sender@mediagoblin.example.org" assert message['To'] == "amanda@example.org, akila@example.org" assert message['Subject'] == "Testing is so much fun!" - assert message.get_payload(decode=True) == """HAYYY GUYS! + assert message.get_payload(decode=True) == b"""HAYYY GUYS! I hope you like unit tests JUST AS MUCH AS I DO!""" @@ -67,7 +67,7 @@ I hope you like unit tests JUST AS MUCH AS I DO!""" assert mbox_message['From'] == "sender@mediagoblin.example.org" assert mbox_message['To'] == "amanda@example.org, akila@example.org" assert mbox_message['Subject'] == "Testing is so much fun!" - assert mbox_message.get_payload(decode=True) == """HAYYY GUYS! + assert mbox_message.get_payload(decode=True) == b"""HAYYY GUYS! I hope you like unit tests JUST AS MUCH AS I DO!""" From cda3055bd6d1810b17a83cde991c7e059ef76657 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 7 Aug 2014 13:08:42 +0300 Subject: [PATCH 39/68] Fix another tests. (forgot to commit earlier) --- mediagoblin/media_types/pdf/processing.py | 1 + mediagoblin/submit/lib.py | 2 +- mediagoblin/tests/test_api.py | 6 +++--- mediagoblin/tests/test_auth.py | 4 ++-- mediagoblin/tests/test_edit.py | 3 +-- mediagoblin/tests/test_exif.py | 6 +++--- mediagoblin/tests/test_http_callback.py | 2 +- mediagoblin/tests/test_notifications.py | 4 ++-- mediagoblin/tests/test_oauth2.py | 6 +++--- mediagoblin/tests/test_pdf.py | 5 +++-- mediagoblin/tests/test_piwigo.py | 17 ++++++----------- mediagoblin/tests/test_pluginapi.py | 3 +-- mediagoblin/tools/crypto.py | 2 +- 13 files changed, 28 insertions(+), 33 deletions(-) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index 0a173cf7..e02fd577 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -207,6 +207,7 @@ def pdf_info(original): _log.debug('pdfinfo could not read the pdf file.') raise BadMediaFail() + lines = [l.decode() for l in lines] info_dict = dict([[part.strip() for part in l.strip().split(':', 1)] for l in lines if ':' in l]) diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index 7d3ea8df..61d5067f 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -59,7 +59,7 @@ def get_upload_file_limits(user): """ Get the upload_limit and max_file_size for this user """ - if user.upload_limit >= 0: + if user.upload_limit is not None and user.upload_limit >= 0: # TODO: debug this upload_limit = user.upload_limit else: upload_limit = mg_globals.app_config.get('upload_limit', None) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index e19313e4..c8a254d4 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -48,10 +48,10 @@ class TestAPI(object): return template.TEMPLATE_TEST_CONTEXT[template_name] def http_auth_headers(self): - return {'Authorization': 'Basic {0}'.format( - base64.b64encode(':'.join([ + return {'Authorization': ('Basic {0}'.format( + base64.b64encode((':'.join([ self.user.username, - self.user_password])))} + self.user_password])).encode('ascii')).decode()))} def do_post(self, data, test_app, **kwargs): url = kwargs.pop('url', '/api/submit') diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index 9b7f9825..7980953f 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -119,7 +119,7 @@ def test_register_views(test_app): assert message['To'] == 'angrygrrl@example.org' email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/verification_email.txt'] - assert email_context['verification_url'] in message.get_payload(decode=True) + assert email_context['verification_url'].encode('ascii') in message.get_payload(decode=True) path = urlparse.urlsplit(email_context['verification_url'])[2] get_params = urlparse.urlsplit(email_context['verification_url'])[3] @@ -190,7 +190,7 @@ def test_register_views(test_app): email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/plugins/basic_auth/fp_verification_email.txt'] #TODO - change the name of verification_url to something forgot-password-ish - assert email_context['verification_url'] in message.get_payload(decode=True) + assert email_context['verification_url'].encode('ascii') in message.get_payload(decode=True) path = urlparse.urlsplit(email_context['verification_url'])[2] get_params = urlparse.urlsplit(email_context['verification_url'])[3] diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index b60d4c74..cf72f308 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -142,8 +142,7 @@ class TestUserEdit(object): assert message['To'] == 'new@example.com' email_context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/edit/verification.txt'] - assert email_context['verification_url'] in \ - message.get_payload(decode=True) + assert email_context['verification_url'].encode('ascii') in message.get_payload(decode=True) path = urlparse.urlsplit(email_context['verification_url'])[2] assert path == u'/edit/verify_email/' diff --git a/mediagoblin/tests/test_exif.py b/mediagoblin/tests/test_exif.py index 861b768a..ccc91d03 100644 --- a/mediagoblin/tests/test_exif.py +++ b/mediagoblin/tests/test_exif.py @@ -54,19 +54,19 @@ def test_exif_extraction(): expected = OrderedDict({'EXIF CVAPattern': {'field_length': 8, 'field_offset': 26224, 'field_type': 7, - 'printable': u'[0, 2, 0, 2, 1, 2, 0, 1]', + 'printable': '[0, 2, 0, 2, 1, 2, 0, 1]', 'tag': 41730, 'values': [0, 2, 0, 2, 1, 2, 0, 1]}, 'EXIF ColorSpace': {'field_length': 2, 'field_offset': 476, 'field_type': 3, - 'printable': u'sRGB', + 'printable': 'sRGB', 'tag': 40961, 'values': [1]}, 'EXIF ComponentsConfiguration': {'field_length': 4, 'field_offset': 308, 'field_type': 7, - 'printable': u'YCbCr', + 'printable': 'YCbCr', 'tag': 37121, 'values': [1, 2, 3, 0]}, 'EXIF CompressedBitsPerPixel': {'field_length': 8, diff --git a/mediagoblin/tests/test_http_callback.py b/mediagoblin/tests/test_http_callback.py index 11f02c97..38f1cfaf 100644 --- a/mediagoblin/tests/test_http_callback.py +++ b/mediagoblin/tests/test_http_callback.py @@ -51,7 +51,7 @@ class TestHTTPCallback(object): 'client_id': client_id, 'client_secret': client_secret}) - response_data = json.loads(response.body) + response_data = json.loads(response.body.decode()) return response_data['access_token'] diff --git a/mediagoblin/tests/test_notifications.py b/mediagoblin/tests/test_notifications.py index 37d61c41..385da569 100644 --- a/mediagoblin/tests/test_notifications.py +++ b/mediagoblin/tests/test_notifications.py @@ -135,13 +135,13 @@ otherperson@example.com\n\nSGkgb3RoZXJwZXJzb24sCmNocmlzIGNvbW1lbnRlZCBvbiB5b3VyI self.logout() self.login('otherperson', 'nosreprehto') - self.test_app.get(media_uri_slug + '/c/{0}/'.format(comment_id)) + self.test_app.get(media_uri_slug + 'c/{0}/'.format(comment_id)) notification = Notification.query.filter_by(id=notification_id).first() assert notification.seen == True - self.test_app.get(media_uri_slug + '/notifications/silence/') + self.test_app.get(media_uri_slug + 'notifications/silence/') subscription = CommentSubscription.query.filter_by(id=subscription_id)\ .first() diff --git a/mediagoblin/tests/test_oauth2.py b/mediagoblin/tests/test_oauth2.py index 014808a6..16372730 100644 --- a/mediagoblin/tests/test_oauth2.py +++ b/mediagoblin/tests/test_oauth2.py @@ -163,7 +163,7 @@ code={1}&client_secret={2}'.format(client_id, code, client.secret)) assert token_res.status_int == 200 - token_data = json.loads(token_res.body) + token_data = json.loads(token_res.body.decode()) assert not 'error' in token_data assert 'access_token' in token_data @@ -191,7 +191,7 @@ code={0}&client_secret={1}'.format(code, client.secret)) assert token_res.status_int == 200 - token_data = json.loads(token_res.body) + token_data = json.loads(token_res.body.decode()) assert 'error' in token_data assert not 'access_token' in token_data @@ -215,7 +215,7 @@ code={0}&client_secret={1}'.format(code, client.secret)) assert token_res.status_int == 200 - new_token_data = json.loads(token_res.body) + new_token_data = json.loads(token_res.body.decode()) assert not 'error' in new_token_data assert 'access_token' in new_token_data diff --git a/mediagoblin/tests/test_pdf.py b/mediagoblin/tests/test_pdf.py index b4d1940a..7e59de17 100644 --- a/mediagoblin/tests/test_pdf.py +++ b/mediagoblin/tests/test_pdf.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import collections import tempfile import shutil import os @@ -26,13 +27,13 @@ from .resources import GOOD_PDF as GOOD @pytest.mark.skipif("not check_prerequisites()") def test_pdf(): - good_dict = {'pdf_version_major': 1, 'pdf_title': '', + good_dict = collections.OrderedDict({'pdf_version_major': 1, 'pdf_title': '', 'pdf_page_size_width': 612, 'pdf_author': '', 'pdf_keywords': '', 'pdf_pages': 10, 'pdf_producer': 'dvips + GNU Ghostscript 7.05', 'pdf_version_minor': 3, 'pdf_creator': 'LaTeX with hyperref package', - 'pdf_page_size_height': 792} + 'pdf_page_size_height': 792}) assert pdf_info(GOOD) == good_dict temp_dir = tempfile.mkdtemp() create_pdf_thumb(GOOD, os.path.join(temp_dir, 'good_256_256.png'), 256, 256) diff --git a/mediagoblin/tests/test_piwigo.py b/mediagoblin/tests/test_piwigo.py index 16ad0111..33aea580 100644 --- a/mediagoblin/tests/test_piwigo.py +++ b/mediagoblin/tests/test_piwigo.py @@ -44,28 +44,23 @@ class Test_PWG(object): def test_session(self): resp = self.do_post("pwg.session.login", {"username": u"nouser", "password": "wrong"}) - assert resp.body == XML_PREFIX \ - + '' + assert resp.body == (XML_PREFIX + '').encode('ascii') resp = self.do_post("pwg.session.login", {"username": self.username, "password": "wrong"}) - assert resp.body == XML_PREFIX \ - + '' + assert resp.body == (XML_PREFIX + '').encode('ascii') resp = self.do_get("pwg.session.getStatus") - assert resp.body == XML_PREFIX \ - + 'guest' + assert resp.body == (XML_PREFIX + 'guest').encode('ascii') resp = self.do_post("pwg.session.login", {"username": self.username, "password": self.password}) - assert resp.body == XML_PREFIX + '1' + assert resp.body == (XML_PREFIX + '1').encode('ascii') resp = self.do_get("pwg.session.getStatus") - assert resp.body == XML_PREFIX \ - + 'chris' + assert resp.body == (XML_PREFIX + 'chris').encode('ascii') self.do_get("pwg.session.logout") resp = self.do_get("pwg.session.getStatus") - assert resp.body == XML_PREFIX \ - + 'guest' + assert resp.body == (XML_PREFIX + 'guest').encode('ascii') diff --git a/mediagoblin/tests/test_pluginapi.py b/mediagoblin/tests/test_pluginapi.py index b3f37e2a..2fd6df39 100644 --- a/mediagoblin/tests/test_pluginapi.py +++ b/mediagoblin/tests/test_pluginapi.py @@ -456,11 +456,10 @@ def test_plugin_staticdirect(static_plugin_app): Test that the staticdirect utilities pull up the right things """ result = json.loads( - static_plugin_app.get('/staticstuff/').body) + static_plugin_app.get('/staticstuff/').body.decode()) assert len(result) == 2 assert result['mgoblin_bunny_pic'] == '/test_static/images/bunny_pic.png' assert result['plugin_bunny_css'] == \ '/plugin_static/staticstuff/css/bunnify.css' - diff --git a/mediagoblin/tools/crypto.py b/mediagoblin/tools/crypto.py index 14a1a72d..cf6d31f6 100644 --- a/mediagoblin/tools/crypto.py +++ b/mediagoblin/tools/crypto.py @@ -61,7 +61,7 @@ def create_key(key_dir, key_filepath): key = str(getrandbits(192)) key_file = tempfile.NamedTemporaryFile(dir=key_dir, suffix='.bin', delete=False) - key_file.write(key) + key_file.write(key.encode('ascii')) key_file.flush() os.rename(key_file.name, key_filepath) key_file.close() From aaa12d637ebdd6e6900b8f0ef027e5fcd595c55a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 7 Aug 2014 13:12:38 +0300 Subject: [PATCH 40/68] Update TODOs in setup.py. --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f9c7f70a..401d55d8 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,9 @@ if PY2: py2_only_install_requires.append('mock') # mock is in the stdlib for 3.3+ install_requires = [ - 'gunicorn==19', # TODO: Upgrade to 19.2 -- see https://github.com/benoitc/gunicorn/issues/830 + # TODO(berker): Upgrade to 19.2 + # See https://github.com/benoitc/gunicorn/issues/830 + 'gunicorn==19', 'python-dateutil', 'wtforms', 'py-bcrypt', @@ -76,7 +78,7 @@ install_requires = [ 'six==1.5.2', 'oauthlib>=0.5.0', 'unidecode', - 'ExifRead', + 'ExifRead', # TODO(berker): Install develop branch for Python 3 'PasteDeploy', # This is optional: # 'translitcodec', From 4930c2adbd19238ffb1da45509bd5e0c5d85ce66 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 13 Aug 2014 19:27:23 +0300 Subject: [PATCH 41/68] Fix a comment. --- mediagoblin/gmg_commands/dbupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 27283a20..8fad99b4 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -116,7 +116,7 @@ def run_dbupdate(app_config, global_config): # Set up the database db = setup_connection_and_db_from_config(app_config, migrations=True) - #Run the migrations + # Run the migrations run_all_migrations(db, app_config, global_config) From 7df0793441b670c722310d038938d44748a925ec Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 13 Aug 2014 19:27:49 +0300 Subject: [PATCH 42/68] Fix dict.keys() in Python 3. --- mediagoblin/gmg_commands/dbupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 8fad99b4..f0a7739e 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -131,7 +131,7 @@ def run_all_migrations(db, app_config, global_config): """ # Gather information from all media managers / projects dbdatas = gather_database_data( - global_config.get('plugins', {}).keys()) + list(global_config.get('plugins', {}).keys())) Session = sessionmaker(bind=db.engine) From 65f20ca43592c2e8beca9b04651d9d1f6aa6b202 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 13 Aug 2014 19:30:23 +0300 Subject: [PATCH 43/68] Add initial Alembic migrations. --- alembic.ini | 59 +++++++++++++++++ mediagoblin/db/migration_tools.py | 29 +++++++++ mediagoblin/db/migrations/README | 1 + mediagoblin/db/migrations/env.py | 71 +++++++++++++++++++++ mediagoblin/db/migrations/script.py.mako | 22 +++++++ mediagoblin/db/migrations/versions/.gitkeep | 0 mediagoblin/gmg_commands/dbupdate.py | 8 +++ setup.py | 1 + 8 files changed, 191 insertions(+) create mode 100644 alembic.ini create mode 100644 mediagoblin/db/migrations/README create mode 100644 mediagoblin/db/migrations/env.py create mode 100644 mediagoblin/db/migrations/script.py.mako create mode 100644 mediagoblin/db/migrations/versions/.gitkeep diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..2fa08099 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,59 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/mediagoblin/db/migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +sqlalchemy.url = sqlite:///mediagoblin.db + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index 1967dacd..2d7b999a 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -16,6 +16,12 @@ from __future__ import unicode_literals +import os + +from alembic import command +from alembic.config import Config + +from mediagoblin.db.base import Base from mediagoblin.tools.common import simple_printer from sqlalchemy import Table from sqlalchemy.sql import select @@ -24,6 +30,29 @@ class TableAlreadyExists(Exception): pass +class AlembicMigrationManager(object): + + def __init__(self, session): + root_dir = os.path.abspath(os.path.dirname(os.path.dirname( + os.path.dirname(__file__)))) + alembic_cfg_path = os.path.join(root_dir, 'alembic.ini') + self.alembic_cfg = Config(alembic_cfg_path) + self.session = session + + def init_tables(self): + Base.metadata.create_all(self.session.bind) + # load the Alembic configuration and generate the + # version table, "stamping" it with the most recent rev: + command.stamp(self.alembic_cfg, 'head') + + def init_or_migrate(self, version='head'): + # TODO(berker): Check this + # http://alembic.readthedocs.org/en/latest/api.html#alembic.migration.MigrationContext + # current_rev = context.get_current_revision() + # Call self.init_tables() first if current_rev is None? + command.upgrade(self.alembic_cfg, version) + + class MigrationManager(object): """ Migration handling tool. diff --git a/mediagoblin/db/migrations/README b/mediagoblin/db/migrations/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/mediagoblin/db/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/mediagoblin/db/migrations/env.py b/mediagoblin/db/migrations/env.py new file mode 100644 index 00000000..712b6164 --- /dev/null +++ b/mediagoblin/db/migrations/env.py @@ -0,0 +1,71 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() + diff --git a/mediagoblin/db/migrations/script.py.mako b/mediagoblin/db/migrations/script.py.mako new file mode 100644 index 00000000..95702017 --- /dev/null +++ b/mediagoblin/db/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/mediagoblin/db/migrations/versions/.gitkeep b/mediagoblin/db/migrations/versions/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index f0a7739e..7039b95c 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -106,6 +106,13 @@ forgotten to add it? ({1})'.format(plugin, exc)) return managed_dbdata +def run_alembic_migrations(db, app_config, global_config): + from mediagoblin.db.migration_tools import AlembicMigrationManager + Session = sessionmaker(bind=db.engine) + manager = AlembicMigrationManager(Session()) + manager.init_or_migrate() + + def run_dbupdate(app_config, global_config): """ Initialize or migrate the database as specified by the config file. @@ -118,6 +125,7 @@ def run_dbupdate(app_config, global_config): db = setup_connection_and_db_from_config(app_config, migrations=True) # Run the migrations run_all_migrations(db, app_config, global_config) + run_alembic_migrations(db, app_config, global_config) def run_all_migrations(db, app_config, global_config): diff --git a/setup.py b/setup.py index 401d55d8..fd089a5a 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ install_requires = [ # TODO(berker): Upgrade to 19.2 # See https://github.com/benoitc/gunicorn/issues/830 'gunicorn==19', + 'alembic==0.6.6', 'python-dateutil', 'wtforms', 'py-bcrypt', From de51eca53f1e7fbbc2175caaf3428c027538c954 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 15:39:45 +0300 Subject: [PATCH 44/68] Provide a better manager API for Alembic. --- mediagoblin/db/migration_tools.py | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index 2d7b999a..ab4487d2 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -16,16 +16,21 @@ from __future__ import unicode_literals +import logging import os from alembic import command from alembic.config import Config +from alembic.migration import MigrationContext from mediagoblin.db.base import Base from mediagoblin.tools.common import simple_printer from sqlalchemy import Table from sqlalchemy.sql import select +log = logging.getLogger(__name__) + + class TableAlreadyExists(Exception): pass @@ -39,18 +44,34 @@ class AlembicMigrationManager(object): self.alembic_cfg = Config(alembic_cfg_path) self.session = session + def get_current_revision(self): + context = MigrationContext.configure(self.session.bind) + return context.get_current_revision() + + def upgrade(self, version): + return command.upgrade(self.alembic_cfg, version or 'head') + + def downgrade(self, version): + if isinstance(version, int) or version is None or version.isdigit(): + version = 'base' + return command.downgrade(self.alembic_cfg, version) + + def stamp(self, revision): + return command.stamp(self.alembic_cfg, revision=revision) + def init_tables(self): Base.metadata.create_all(self.session.bind) # load the Alembic configuration and generate the # version table, "stamping" it with the most recent rev: command.stamp(self.alembic_cfg, 'head') - def init_or_migrate(self, version='head'): - # TODO(berker): Check this - # http://alembic.readthedocs.org/en/latest/api.html#alembic.migration.MigrationContext - # current_rev = context.get_current_revision() - # Call self.init_tables() first if current_rev is None? - command.upgrade(self.alembic_cfg, version) + def init_or_migrate(self, version=None): + if self.get_current_revision() is None: + log.info('Initializing tables and stamping it with ' + 'the most recent migration...') + self.init_tables() + else: + self.upgrade(version) class MigrationManager(object): From 2064ad9450f6260eaf583a205d865d167380d59d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 15:40:35 +0300 Subject: [PATCH 45/68] Move the AlembicMigrationManager to top of the module. Also, add a simple docstring. --- mediagoblin/gmg_commands/dbupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 7039b95c..f5c20720 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -19,7 +19,7 @@ import logging from sqlalchemy.orm import sessionmaker from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.db.migration_tools import MigrationManager +from mediagoblin.db.migration_tools import MigrationManager, AlembicMigrationManager from mediagoblin.init import setup_global_and_app_config from mediagoblin.tools.common import import_component @@ -107,7 +107,7 @@ forgotten to add it? ({1})'.format(plugin, exc)) def run_alembic_migrations(db, app_config, global_config): - from mediagoblin.db.migration_tools import AlembicMigrationManager + """Initializes a database and runs all Alembic migrations.""" Session = sessionmaker(bind=db.engine) manager = AlembicMigrationManager(Session()) manager.init_or_migrate() From 3dbdb061ea810718ecb8921bb7e3444423b9d211 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 18:57:48 +0300 Subject: [PATCH 46/68] Improve mediagoblin._compat.py2_unicode. - Encode obj.__repr__() to bytestring if its type is unicode in Python 2. - Add internal encode_to_utf8() decorator. - Do not raise an exception if a class does not have an __str__() method, just warn. --- mediagoblin/_compat.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/mediagoblin/_compat.py b/mediagoblin/_compat.py index 5a3fac53..9164d5fc 100644 --- a/mediagoblin/_compat.py +++ b/mediagoblin/_compat.py @@ -1,20 +1,32 @@ -from six import PY3 +import functools +import warnings -if PY3: +import six + +if six.PY3: from email.mime.text import MIMEText else: from email.MIMEText import MIMEText -# taken from -# https://github.com/django/django/blob/master/django/utils/encoding.py +def encode_to_utf8(method): + def wrapper(self): + if six.PY2 and isinstance(method(self), six.text_type): + return method(self).encode('utf-8') + return method(self) + functools.update_wrapper(wrapper, method, ['__name__', '__doc__']) + return wrapper + + +# based on django.utils.encoding.python_2_unicode_compatible def py2_unicode(klass): - # TODO: Add support for __repr__ - if not PY3: + if six.PY2: if '__str__' not in klass.__dict__: - raise ValueError("@py2_unicode cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) + warnings.warn("@py2_unicode cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + klass.__str__ = encode_to_utf8(klass.__unicode__) + if '__repr__' in klass.__dict__: + klass.__repr__ = encode_to_utf8(klass.__repr__) return klass From f3c1b6ff6d0faebd6eff272849babe26851b423b Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 22:35:22 +0300 Subject: [PATCH 47/68] Restore different server options. --- paste.ini | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/paste.ini b/paste.ini index b859251e..0cb01408 100644 --- a/paste.ini +++ b/paste.ini @@ -59,3 +59,27 @@ port = 6543 proc_name = gmg reload = true accesslog = - + +####################### +# Helper server configs +# --------------------- +# If you are configuring the paste config manually, you can remove +# these. + +# Use this if you want to run on port 6543 and have MediaGoblin be +# viewable externally +[server:broadcast] +use = egg:Paste#http +host = 0.0.0.0 +port = 6543 + +# Use this if you want to connect via fastcgi +[server:fcgi] +use = egg:flup#fcgi_fork +host = %(fcgi_host)s +port = %(fcgi_port)s + +[server:http] +use = egg:Paste#http +host = %(http_host)s +port = %(http_port)s From 051c728c6efa21493fd85bb7d07a8acf17a40e42 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 23:02:22 +0300 Subject: [PATCH 48/68] Add an initial "gmg serve" implementation. The CLI is similar to "paster serve". --- mediagoblin/gmg_commands/__init__.py | 4 ++ mediagoblin/gmg_commands/serve.py | 63 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 mediagoblin/gmg_commands/serve.py diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index e0400897..79fb7701 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -59,6 +59,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.deletemedia:parser_setup', 'func': 'mediagoblin.gmg_commands.deletemedia:deletemedia', 'help': 'Delete media entries'}, + 'serve': { + 'setup': 'mediagoblin.gmg_commands.serve:parser_setup', + 'func': 'mediagoblin.gmg_commands.serve:serve', + 'help': 'PasteScript replacement'}, # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/serve.py b/mediagoblin/gmg_commands/serve.py new file mode 100644 index 00000000..44f54697 --- /dev/null +++ b/mediagoblin/gmg_commands/serve.py @@ -0,0 +1,63 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import print_function + + +class ServeCommand(object): + + def loadserver(self, server_spec, name, relative_to, **kw): + pass + + def loadapp(self, app_spec, name, relative_to, **kw): + pass + + def daemonize(self): + # TODO: pass to gunicorn if available + pass + + def restart_with_reloader(self): + pass + + def restart_with_monitor(self, reloader=False): + pass + + def run(self): + print('Running...') + + +def parser_setup(subparser): + subparser.add_argument('config', metavar='CONFIG_FILE') + subparser.add_argument('command', + choices=['start', 'stop', 'restart', 'status'], + nargs='?', default='start') + subparser.add_argument('-n', '--app-name', + dest='app_name', + metavar='NAME', + help="Load the named application (default main)") + subparser.add_argument('-s', '--server', + dest='server', + metavar='SERVER_TYPE', + help="Use the named server.") + subparser.add_argument('--reload', + dest='reload', + action='store_true', + help="Use auto-restart file monitor") + + +def serve(args): + serve_cmd = ServeCommand() # TODO: pass args to it + serve_cmd.run() From 39a9035487621ab079138149a0dd090ac9ab5768 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 15 Aug 2014 23:07:35 +0300 Subject: [PATCH 49/68] Implement ServeCommand.{loadserver, loadapp}. This is partly port from PasteScript. --- mediagoblin/gmg_commands/serve.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mediagoblin/gmg_commands/serve.py b/mediagoblin/gmg_commands/serve.py index 44f54697..64400fdd 100644 --- a/mediagoblin/gmg_commands/serve.py +++ b/mediagoblin/gmg_commands/serve.py @@ -16,14 +16,17 @@ from __future__ import print_function +from paste.deploy import loadapp, loadserver + class ServeCommand(object): - def loadserver(self, server_spec, name, relative_to, **kw): - pass + def loadserver(self, server_spec, name, relative_to, **kwargs): + return loadserver(server_spec, name=name, relative_to=relative_to, + **kwargs) - def loadapp(self, app_spec, name, relative_to, **kw): - pass + def loadapp(self, app_spec, name, relative_to, **kwargs): + return loadapp(app_spec, name=name, relative_to=relative_to, **kwargs) def daemonize(self): # TODO: pass to gunicorn if available From b6df960806ccb091fc334bcab269be69eec339d8 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Thu, 11 Sep 2014 15:34:19 -0500 Subject: [PATCH 50/68] Set up tox.ini to run more properly: - Don't run dbupdate... the tests themselves do this, and we might mess up someone's db - We shouldn't run setup.py develop because that installs a new *live* db... the tests do isolation, so... - Install the package's dependencies by the package itself... it seems that removing "skipsdist = True" fixes this - Run py.test manually rather than use runtests.sh (which itself uses ./bin/py.test if it can) --- tox.ini | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 4f123c1b..fa6d0b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,11 @@ [tox] envlist = py27, py33 -skipsdist = True usedevelop = True +sitepackages = False [testenv] whitelist_externals = sh -commands = - python setup.py develop - gmg dbupdate - sh runtests.sh +commands = py.test ./mediagoblin/tests --boxed deps = git+https://github.com/ianare/exif-py.git@develop lxml From 99c466045a1e5eb9032f4938729435c5d6a6e1aa Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 12 Sep 2014 09:30:46 -0500 Subject: [PATCH 51/68] Fix unicode error in pdf media type (we're checking against bytestrings, so make that explicit) --- mediagoblin/media_types/pdf/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index e02fd577..08a00019 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -141,7 +141,7 @@ def is_unoconv_working(): except OSError: _log.warn(_('unoconv failing to run, check log file')) return False - if 'ERROR' in output: + if b'ERROR' in output: return False return True From 9156ab68d0cb251bd47ab832912bfc59156653fc Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 12 Sep 2014 10:42:23 -0500 Subject: [PATCH 52/68] Explicitly open READMEFILE as utf-8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fd089a5a..0c182e98 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ install_requires = [ # 'Pillow', ] + py2_only_install_requires -with open(READMEFILE) as fobj: +with open(READMEFILE, encoding="utf-8") as fobj: long_description = fobj.read() try: From 0c32c7fe8338d2689d7afafa5164f03fa440bd47 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 12 Sep 2014 12:29:12 -0500 Subject: [PATCH 53/68] py2.7 compatibility with open(..., encoding="utf-8"), use io.open This commit sponsored by Peter Baumgarten. Thank you! --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0c182e98..0d4af5ab 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ from __future__ import print_function from setuptools import setup, find_packages +from io import open import os import re From 3c06c3efafaf6834e157f3ffa17eb1f0b42f5a39 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 15 Sep 2014 11:46:56 -0500 Subject: [PATCH 54/68] Upgrading gunicorn. Referenced issue is resolved. Also, upgrading seems to fix weird "RuntimeError: dictionary changed size during iteration" issues I was seeing. (Looked like a gunicorn issue, not ours.) --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0d4af5ab..ed10df02 100644 --- a/setup.py +++ b/setup.py @@ -55,9 +55,7 @@ if PY2: py2_only_install_requires.append('mock') # mock is in the stdlib for 3.3+ install_requires = [ - # TODO(berker): Upgrade to 19.2 - # See https://github.com/benoitc/gunicorn/issues/830 - 'gunicorn==19', + 'gunicorn', 'alembic==0.6.6', 'python-dateutil', 'wtforms', From 0977d10d2a3435230fc3dad0659b613d96a362e7 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 15 Sep 2014 12:10:29 -0500 Subject: [PATCH 55/68] No reason to wrap the rest of the options in a quote This commit sponsored by Thane Williams. Thank you! --- lazystarter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lazystarter.sh b/lazystarter.sh index 2842d5d5..102830ad 100755 --- a/lazystarter.sh +++ b/lazystarter.sh @@ -70,7 +70,7 @@ set -x export CELERY_ALWAYS_EAGER=true case "$selfname" in lazyserver.sh) - $starter --paste "$ini_file" "$@" + $starter --paste "$ini_file" $@ ;; lazycelery.sh) MEDIAGOBLIN_CONFIG="${ini_file}" \ From 5b64c92e0816e733c2f88b88ddc0aec070cdc0d3 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 15 Sep 2014 14:26:57 -0500 Subject: [PATCH 56/68] Temporarily disabling the mediagoblin errors middleware We don't have paste (core) in py3, so.... --- paste.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paste.ini b/paste.ini index 0cb01408..afd5982b 100644 --- a/paste.ini +++ b/paste.ini @@ -6,7 +6,8 @@ debug = false [pipeline:main] -pipeline = errors mediagoblin +# pipeline = errors mediagoblin +pipeline = mediagoblin [app:mediagoblin] use = egg:mediagoblin#app From 3a02813c7a70bb2e919101343a0680fe1aa34a81 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 14:26:40 -0500 Subject: [PATCH 57/68] Import mock from unittest if on py3 --- mediagoblin/tests/test_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 329c387d..31bf50b3 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -15,7 +15,10 @@ # along with this program. If not, see . import json -import mock +try: + import mock +except ImportError: + import unittest.mock as mock import pytest from webtest import AppError From 58a7292fed40cfad3cb386f016505d2e0ec43ad2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 14:37:36 -0500 Subject: [PATCH 58/68] Fix test_legacy_api.py Or rather, reimplement one of Berker's fixes and add one of mine: - add back the http_auth_headers fix Berker wrote - decode to json when testing the response.body, since we have no idea what the order will be here --- mediagoblin/tests/test_legacy_api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mediagoblin/tests/test_legacy_api.py b/mediagoblin/tests/test_legacy_api.py index 4e0cbd8f..f12d9ce5 100644 --- a/mediagoblin/tests/test_legacy_api.py +++ b/mediagoblin/tests/test_legacy_api.py @@ -17,6 +17,7 @@ import logging import base64 +import json import pytest @@ -48,10 +49,10 @@ class TestAPI(object): return template.TEMPLATE_TEST_CONTEXT[template_name] def http_auth_headers(self): - return {'Authorization': 'Basic {0}'.format( - base64.b64encode(':'.join([ + return {'Authorization': ('Basic {0}'.format( + base64.b64encode((':'.join([ self.user.username, - self.user_password])))} + self.user_password])).encode('ascii')).decode()))} def do_post(self, data, test_app, **kwargs): url = kwargs.pop('url', '/api/submit') @@ -77,8 +78,8 @@ class TestAPI(object): '/api/test', headers=self.http_auth_headers()) - assert response.body == \ - '{"username": "joapi", "email": "joapi@example.com"}' + assert json.loads(response.body) == { + "username": "joapi", "email": "joapi@example.com"} def test_2_test_submission(self, test_app): self.login(test_app) From b6774d339af5344ead3b1d11b72e365fee01fbb1 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 14:37:52 -0500 Subject: [PATCH 59/68] Simplify things here and use io.open --- mediagoblin/tools/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index bfefcac9..aeb4f829 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -15,6 +15,7 @@ # along with this program. If not, see . +from io import open import os import copy import json @@ -102,7 +103,7 @@ def load_resource(package, resource_path): os.path.sep. """ filename = resource_filename(package, os.path.sep.join(resource_path)) - return file(filename).read() + return open(filename, encoding="utf-8").read() def load_resource_json(package, resource_path): """ From fa3f46d7144629eed6f1f6f9abc884a1b8f4c330 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 14:46:02 -0500 Subject: [PATCH 60/68] Import mock correctly on py3 This commit sponsored by Andrew McNicol. Thank you! --- mediagoblin/tests/test_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index e1c3c7e5..8193233f 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -14,7 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import mock +try: + import mock +except ImportError: + import unittest.mock as mock import email import pytest import smtplib From 21cbf8294eba59b29d56f30ce32b0cc1f9587981 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 14:56:13 -0500 Subject: [PATCH 61/68] json.loads(request.body) => json.loads(response.body.decode())) This fixes python 3 stuff. This commit sponsored by James Reilly. Thanks, James! --- mediagoblin/tests/test_api.py | 14 +++++++------- mediagoblin/tests/test_legacy_api.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 31bf50b3..3ac21366 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -58,7 +58,7 @@ class TestAPI(object): headers=headers ) - return response, json.loads(response.body) + return response, json.loads(response.body.decode()) def _upload_image(self, test_app, image): """ Uploads and image to MediaGoblin via pump.io API """ @@ -75,7 +75,7 @@ class TestAPI(object): data, headers=headers ) - image = json.loads(response.body) + image = json.loads(response.body.decode()) return response, image @@ -227,7 +227,7 @@ class TestAPI(object): headers={"Content-Type": "application/json"} ) - image = json.loads(response.body)["object"] + image = json.loads(response.body.decode())["object"] # Check everything has been set on the media correctly media = MediaEntry.query.filter_by(id=image["id"]).first() @@ -406,7 +406,7 @@ class TestAPI(object): uri = "/api/user/{0}/profile".format(self.user.username) with self.mock_oauth(): response = test_app.get(uri) - profile = json.loads(response.body) + profile = json.loads(response.body.decode()) assert response.status_code == 200 @@ -420,7 +420,7 @@ class TestAPI(object): uri = "/api/user/{0}/".format(self.user.username) with self.mock_oauth(): response = test_app.get(uri) - user = json.loads(response.body) + user = json.loads(response.body.decode()) assert response.status_code == 200 @@ -446,7 +446,7 @@ class TestAPI(object): uri = "/api/user/{0}/feed".format(self.active_user.username) with self.mock_oauth(): response = test_app.get(uri) - feed = json.loads(response.body) + feed = json.loads(response.body.decode()) assert response.status_code == 200 @@ -481,7 +481,7 @@ class TestAPI(object): with self.mock_oauth(): response = test_app.get(data["object"]["links"]["self"]["href"]) - data = json.loads(response.body) + data = json.loads(response.body.decode()) assert response.status_code == 200 diff --git a/mediagoblin/tests/test_legacy_api.py b/mediagoblin/tests/test_legacy_api.py index f12d9ce5..b3b2fcec 100644 --- a/mediagoblin/tests/test_legacy_api.py +++ b/mediagoblin/tests/test_legacy_api.py @@ -78,7 +78,7 @@ class TestAPI(object): '/api/test', headers=self.http_auth_headers()) - assert json.loads(response.body) == { + assert json.loads(response.body.decode()) == { "username": "joapi", "email": "joapi@example.com"} def test_2_test_submission(self, test_app): From 13f37e75eb9a1f3e68b183eacfd1a51572d452c2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:07:33 -0500 Subject: [PATCH 62/68] Fix iteritems usage on python 3 This commit sponsored by Ben (Free Software Melbourne) Finney. Thanks! --- mediagoblin/edit/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 59d851f6..7359f520 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -455,7 +455,7 @@ def edit_metadata(request, media): return redirect_obj(request, media) if len(form.media_metadata) == 0: - for identifier, value in media.media_metadata.iteritems(): + for identifier, value in six.iteritems(media.media_metadata): if identifier == "@context": continue form.media_metadata.append_entry({ 'identifier':identifier, From 7893d43a8f7d3e42b21491444156c4a5308e211d Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:08:28 -0500 Subject: [PATCH 63/68] Fix exception catching on python 3 This commit sponsored by Paul Smith. Thank you! --- mediagoblin/tests/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tests/test_metadata.py b/mediagoblin/tests/test_metadata.py index b4ea646e..a10e00ec 100644 --- a/mediagoblin/tests/test_metadata.py +++ b/mediagoblin/tests/test_metadata.py @@ -56,7 +56,7 @@ class TestMetadataFunctionality: jsonld_fail_1 = None try: jsonld_fail_1 = compact_and_validate(metadata_fail_1) - except ValidationError, e: + except ValidationError as e: assert e.message == "'All Rights Reserved.' is not a 'uri'" assert jsonld_fail_1 == None #,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,., @@ -72,7 +72,7 @@ class TestMetadataFunctionality: jsonld_fail_2 = None try: jsonld_fail_2 = compact_and_validate(metadata_fail_2) - except ValidationError, e: + except ValidationError as e: assert e.message == "'The other day' is not a 'date-time'" assert jsonld_fail_2 == None From 16450dada24dec2819c6067768af54b0d9b91402 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:15:13 -0500 Subject: [PATCH 64/68] Fix error check in test_edit.py for python 3 This isn't the nicest of checks... we should probably be checking the actual form passed into the context. But for now, it's a fix. --- mediagoblin/tests/test_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index 54f43d68..e34ca27a 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -250,5 +250,5 @@ class TestMetaDataEdit: old_metadata = new_metadata new_metadata = media_entry.media_metadata assert new_metadata == old_metadata - assert ("u'On the worst day' is not a 'date-time'" in + assert (b"'On the worst day' is not a 'date-time'" in response.body) From dd41141d238c6b9329c5250429cec50f0e174246 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:25:30 -0500 Subject: [PATCH 65/68] Much more nicely formed form error check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This doesn't rely on checking HTML output... thus, cleaner. This commit sponsored by Alexandre Guédon. Thank you! --- mediagoblin/tests/test_edit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index e34ca27a..547b382e 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -250,5 +250,7 @@ class TestMetaDataEdit: old_metadata = new_metadata new_metadata = media_entry.media_metadata assert new_metadata == old_metadata - assert (b"'On the worst day' is not a 'date-time'" in - response.body) + context = template.TEMPLATE_TEST_CONTEXT[ + 'mediagoblin/edit/metadata.html'] + assert context['form'].errors['media_metadata'][0]['identifier'][0] == \ + "'On the worst day' is not a 'date-time'" From 37865d02dddab827d3d016148274e462db03c9f2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:33:46 -0500 Subject: [PATCH 66/68] decode to unicode before loading in json again, for py3 This commit sponsored by Chris Cormack. Thanks! --- mediagoblin/federation/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 3d6953a7..cff33224 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -139,7 +139,7 @@ def feed_endpoint(request): return json_error("No such 'user' with id '{0}'".format(username), 404) if request.data: - data = json.loads(request.data) + data = json.loads(request.data.decode()) else: data = {"verb": None, "object": {}} From 1db2bd3fe745d0914264406a0090efc6d81244f4 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 15:35:23 -0500 Subject: [PATCH 67/68] Annnnd another json decode fix for py3! On a roll with these! This commit sponsored by Ramana Kumar. Thanks! --- mediagoblin/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 3ac21366..e3a06cf8 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -284,7 +284,7 @@ class TestAPI(object): with self.mock_oauth(): request = test_app.get(object_uri) - image = json.loads(request.body) + image = json.loads(request.body.decode()) entry = MediaEntry.query.filter_by(id=image["id"]).first() assert request.status_code == 200 From 6430ae97eca57f4db4bcef54436df6c2abcd21ad Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Tue, 16 Sep 2014 17:36:52 -0500 Subject: [PATCH 68/68] Last two issues related to the python 3 merge tests: fixed! - Fix the "pulling the error out of excinfo" stuff for py3 - The u"" only gets embedded in the string on py2. This commit sponsored by Jeff Gibson. Thanks, Jeff! :) --- mediagoblin/tests/test_api.py | 16 ++++++++-------- mediagoblin/tests/test_edit.py | 9 +++++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index e3a06cf8..6b0722aa 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -145,7 +145,7 @@ class TestAPI(object): headers=headers ) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_unable_to_post_feed_as_someone_else(self, test_app): """ Tests that can't post an image to someone else's feed """ @@ -168,7 +168,7 @@ class TestAPI(object): headers=headers ) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_only_able_to_update_own_image(self, test_app): """ Test's that the uploader is the only person who can update an image """ @@ -200,7 +200,7 @@ class TestAPI(object): headers=headers ) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_upload_image_with_filename(self, test_app): """ Tests that you can upload an image with filename and description """ @@ -263,7 +263,7 @@ class TestAPI(object): ) # Assert that we've got a 403 - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_object_endpoint(self, test_app): """ Tests that object can be looked up at endpoint """ @@ -354,7 +354,7 @@ class TestAPI(object): headers=headers ) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_unable_to_update_someone_elses_comment(self, test_app): """ Test that you're able to update someoen elses comment. """ @@ -399,7 +399,7 @@ class TestAPI(object): headers=headers ) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_profile(self, test_app): """ Tests profile endpoint """ @@ -436,7 +436,7 @@ class TestAPI(object): with pytest.raises(AppError) as excinfo: response = test_app.get("/api/whoami") - assert "401 UNAUTHORIZED" in excinfo.value.message + assert "401 UNAUTHORIZED" in excinfo.value.args[0] def test_read_feed(self, test_app): """ Test able to read objects from the feed """ @@ -471,7 +471,7 @@ class TestAPI(object): with pytest.raises(AppError) as excinfo: self._post_image_to_feed(test_app, data) - assert "403 FORBIDDEN" in excinfo.value.message + assert "403 FORBIDDEN" in excinfo.value.args[0] def test_object_endpoint_requestable(self, test_app): """ Test that object endpoint can be requested """ diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index 547b382e..384929cb 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import six import six.moves.urllib.parse as urlparse import pytest @@ -252,5 +253,9 @@ class TestMetaDataEdit: assert new_metadata == old_metadata context = template.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/edit/metadata.html'] - assert context['form'].errors['media_metadata'][0]['identifier'][0] == \ - "'On the worst day' is not a 'date-time'" + if six.PY2: + expected = "u'On the worst day' is not a 'date-time'" + else: + expected = "'On the worst day' is not a 'date-time'" + assert context['form'].errors[ + 'media_metadata'][0]['identifier'][0] == expected