diff --git a/.pep8 b/.flake8 similarity index 100% rename from .pep8 rename to .flake8 diff --git a/CHANGES b/CHANGES index 45fcc6c..6477acc 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ Version 0.9.7 (unreleased) -------------------------- * Arabic translation. (#257, thanks @Bashar) * Translations via the DeepL API (#258, thanks @halitcelik) +* Fixed unicode handling in gettext headers (#259, thanks @NotSqrt) Version 0.9.6 diff --git a/rosetta/templatetags/rosetta.py b/rosetta/templatetags/rosetta.py index ba2aab3..dfd5826 100644 --- a/rosetta/templatetags/rosetta.py +++ b/rosetta/templatetags/rosetta.py @@ -1,12 +1,12 @@ -from django import template -from django.utils.safestring import mark_safe -from django.utils.html import escape import re + import six +from django import template +from django.utils.html import escape +from django.utils.safestring import mark_safe from rosetta.access import can_translate - register = template.Library() rx = re.compile(r'(%(\([^\s\)]*\))?[sd]|\{[\w\d_]+?\})') @@ -14,33 +14,45 @@ can_translate = register.filter(can_translate) def format_message(message): - return mark_safe(rx.sub('\\1', escape(message).replace(r'\n', '
\n'))) + return mark_safe( + rx.sub('\\1', escape(message).replace(r'\n', '
\n')) + ) + + format_message = register.filter(format_message) def lines_count(message): return 1 + sum([len(line) / 50 for line in message.split('\n')]) + + lines_count = register.filter(lines_count) def mult(a, b): return int(a) * int(b) + + mult = register.filter(mult) def minus(a, b): try: return int(a) - int(b) - except: + except Exception: return 0 + + minus = register.filter(minus) def gt(a, b): try: return int(a) > int(b) - except: + except Exception: return False + + gt = register.filter(gt) @@ -54,6 +66,8 @@ def do_incr(parser, token): if name not in parser._namedIncrNodes: parser._namedIncrNodes[name] = IncrNode(0) return parser._namedIncrNodes[name] + + do_incr = register.tag('increment', do_incr) @@ -68,4 +82,6 @@ class IncrNode(template.Node): def is_fuzzy(message): return message and hasattr(message, 'flags') and 'fuzzy' in message.flags + + is_fuzzy = register.filter(is_fuzzy) diff --git a/rosetta/tests/tests.py b/rosetta/tests/tests.py index 21b42a9..9dfdf4f 100644 --- a/rosetta/tests/tests.py +++ b/rosetta/tests/tests.py @@ -4,6 +4,7 @@ import os import shutil from urllib.parse import urlencode +import vcr from django import VERSION from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -13,9 +14,6 @@ from django.test import RequestFactory, TestCase, override_settings from django.test.client import Client from django.urls import resolve, reverse from django.utils.encoding import force_bytes - -import six -import vcr from rosetta import views from rosetta.signals import entry_changed, post_save from rosetta.storage import get_storage @@ -138,10 +136,10 @@ class RosettaTestCase(TestCase): self.copy_po_file_from_template('./django.po.issue67.template') # Make sure the plurals string is valid - with open(self.dest_file, 'rb') as f_: + with open(self.dest_file, 'r') as f_: content = f_.read() - self.assertTrue('Hello, world' not in six.text_type(content)) - self.assertTrue('|| n%100>=20) ? 1 : 2)' in six.text_type(content)) + self.assertTrue('Hello, world' not in content) + self.assertTrue('|| n%100>=20) ? 1 : 2)' in content) del content r = self.client.get(self.xx_form_url + '?msg_filter=untranslated') @@ -157,7 +155,7 @@ class RosettaTestCase(TestCase): self.client.post(self.xx_form_url + '?msg_filter=untranslated', data) # Make sure the plurals string is still valid - with open(self.dest_file, 'rb') as f_: + with open(self.dest_file, 'r') as f_: content = f_.read() self.assertTrue('Hello, world' in str(content)) self.assertTrue('|| n%100>=20) ? 1 : 2)' in str(content)) @@ -277,13 +275,13 @@ class RosettaTestCase(TestCase): # this user. with self.settings(ROSETTA_REQUIRES_AUTH=True): r = self.client3.get(self.xx_form_url) - self.assertFalse(r.content) + self.assertFalse(r.content.decode()) self.assertEqual(r.status_code, 302) # When it's not required, we sail through. with self.settings(ROSETTA_REQUIRES_AUTH=False): r = self.client3.get(self.xx_form_url) - self.assertTrue(r.content) + self.assertTrue(r.content.decode()) self.assertEqual(r.status_code, 200) @override_settings(ROSETTA_LANGUAGES=(('fr', 'French'), ('xx', 'Dummy Language'))) @@ -509,13 +507,13 @@ class RosettaTestCase(TestCase): data = {'m_e48f149a8b2e8baa81b816c0edf93890': 'Hello, world'} r = self.client.post(self.xx_form_url + '?filter=untranslated', data) # read the result - with open(self.dest_file, 'rb') as f_: - content = six.text_type(f_.read()) + with open(self.dest_file, 'r') as f_: + content = f_.read() # make sure unicode data was properly converted to ascii self.assertTrue('Hello, world' in content) self.assertTrue('save_header_data@test.com' in content) - self.assertTrue('aeaeae aaaaaaa aaaa uuuu' in content) + self.assertTrue('aéaéaé aàaàaàa aâââ üüüü' in content) def test_24_percent_translation(self): self.copy_po_file_from_template('./django.po.template') @@ -741,12 +739,8 @@ class RosettaTestCase(TestCase): def test_40_issue_155_auto_compile(self): def file_hash(file_string): - if six.PY3: - with open(file_string, encoding="latin-1") as file: - file_content = file.read().encode('utf-8') - else: - with open(file_string) as file: - file_content = file.read() + with open(file_string, encoding="latin-1") as file: + file_content = file.read().encode('utf-8') return hashlib.md5(file_content).hexdigest() def message_hashes(): @@ -886,7 +880,7 @@ class RosettaTestCase(TestCase): # But if the language isn't an option, we get a 404 with self.settings( - ROSETTA_LANGUAGES=[l for l in settings.LANGUAGES if l[0] != 'xx'] + ROSETTA_LANGUAGES=[lang for lang, __ in settings.LANGUAGES if lang != 'xx'] ): view = self._setup_view( view=views.TranslationFormView(), request=request, **kwargs diff --git a/rosetta/views.py b/rosetta/views.py index cf04c49..f7f2c6d 100644 --- a/rosetta/views.py +++ b/rosetta/views.py @@ -2,20 +2,16 @@ import hashlib import os import os.path import re -import unicodedata import zipfile from urllib.parse import urlencode +import six from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import user_passes_test from django.core.paginator import Paginator -from django.http import ( - Http404, - HttpResponse, - HttpResponseRedirect, - JsonResponse -) +from django.http import (Http404, HttpResponse, HttpResponseRedirect, + JsonResponse) from django.urls import reverse from django.utils.decorators import method_decorator from django.utils.encoding import force_bytes @@ -23,8 +19,6 @@ from django.utils.functional import Promise, cached_property from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.generic import TemplateView, View - -import six from polib import pofile from . import get_version as get_rosetta_version @@ -104,7 +98,7 @@ class RosettaFileLevelMixin(RosettaBaseMixin): """ # (Formerly known as "rosetta_i18n_lang_code") lang_id = self.kwargs['lang_id'] - if lang_id not in {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES}: + if lang_id not in {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES}: raise Http404 if not can_translate_language(self.request.user, lang_id): raise Http404 @@ -227,7 +221,8 @@ class TranslationFileListView(RosettaBaseMixin, TemplateView): third_party_apps=third_party_apps, ) po_files = [ - (get_app_name(l), os.path.realpath(l), pofile(l)) for l in po_paths + (get_app_name(lang), os.path.realpath(lang), pofile(lang)) + for lang in po_paths ] po_files.sort(key=lambda app: app[0]) languages.append((language[0], _(language[1]), po_files)) @@ -367,15 +362,11 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView): if file_change and self.po_file_is_writable: try: - self.po_file.metadata['Last-Translator'] = unicodedata.normalize( - 'NFKD', - u"%s %s <%s>" - % ( - getattr(self.request.user, 'first_name', 'Anonymous'), - getattr(self.request.user, 'last_name', 'User'), - getattr(self.request.user, 'email', 'anonymous@user.tld'), - ), - ).encode('ascii', 'ignore') + self.po_file.metadata['Last-Translator'] = "{} {} <{}>".format( + getattr(self.request.user, 'first_name', 'Anonymous'), + getattr(self.request.user, 'last_name', 'User'), + getattr(self.request.user, 'email', 'anonymous@user.tld'), + ) self.po_file.metadata['X-Translated-Using'] = u"django-rosetta %s" % ( get_rosetta_version() ) @@ -412,7 +403,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView): import uwsgi uwsgi.reload() # pretty easy right? - except: + except Exception: pass # we may not be running under uwsgi :P # XXX: It would be nice to add a success message here! except Exception as e: @@ -575,7 +566,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView): """ ref_lang = self._request_request('ref_lang', 'msgid') if ref_lang != 'msgid': - allowed_languages = {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES} + allowed_languages = {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES} if ref_lang not in allowed_languages: raise Http404 return ref_lang diff --git a/tox.ini b/tox.ini index d076491..66f41e0 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,7 @@ deps = goslate vcrpy coverage + pudb [testenv:gettext] basepython = python3 @@ -74,6 +75,6 @@ commands= [testenv:flake8] basepython = python3 -deps = flake8==2.4.1 +deps = flake8==3.9.2 commands= flake8 {toxinidir}/rosetta