Fixed unicode handling in gettext headers. Fixes #259

CHANGES

@@ -6,6 +6,7 @@ Version 0.9.7 (unreleased)
 --------------------------
 * Arabic translation. (#257, thanks @Bashar)
 * Translations via the DeepL API (#258, thanks @halitcelik)
+* Fixed unicode handling in gettext headers (#259, thanks @NotSqrt)
 
 
 Version 0.9.6

rosetta/templatetags/rosetta.py

@@ -1,12 +1,12 @@
-from django import template
-from django.utils.safestring import mark_safe
-from django.utils.html import escape
 import re
+
 import six
+from django import template
+from django.utils.html import escape
+from django.utils.safestring import mark_safe
 
 from rosetta.access import can_translate
 
-
 register = template.Library()
 rx = re.compile(r'(%(\([^\s\)]*\))?[sd]|\{[\w\d_]+?\})')
 
@@ -14,33 +14,45 @@ can_translate = register.filter(can_translate)
 
 
 def format_message(message):
-    return mark_safe(rx.sub('<code>\\1</code>', escape(message).replace(r'\n', '<br />\n')))
+    return mark_safe(
+        rx.sub('<code>\\1</code>', escape(message).replace(r'\n', '<br />\n'))
+    )
+
+
 format_message = register.filter(format_message)
 
 
 def lines_count(message):
     return 1 + sum([len(line) / 50 for line in message.split('\n')])
+
+
 lines_count = register.filter(lines_count)
 
 
 def mult(a, b):
     return int(a) * int(b)
+
+
 mult = register.filter(mult)
 
 
 def minus(a, b):
     try:
         return int(a) - int(b)
-    except:
+    except Exception:
         return 0
+
+
 minus = register.filter(minus)
 
 
 def gt(a, b):
     try:
         return int(a) > int(b)
-    except:
+    except Exception:
         return False
+
+
 gt = register.filter(gt)
 
 
@@ -54,6 +66,8 @@ def do_incr(parser, token):
     if name not in parser._namedIncrNodes:
         parser._namedIncrNodes[name] = IncrNode(0)
     return parser._namedIncrNodes[name]
+
+
 do_incr = register.tag('increment', do_incr)
 
 
@@ -68,4 +82,6 @@ class IncrNode(template.Node):
 
 def is_fuzzy(message):
     return message and hasattr(message, 'flags') and 'fuzzy' in message.flags
+
+
 is_fuzzy = register.filter(is_fuzzy)

rosetta/tests/tests.py

@@ -4,6 +4,7 @@ import os
 import shutil
 from urllib.parse import urlencode
 
+import vcr
 from django import VERSION
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
@@ -13,9 +14,6 @@ from django.test import RequestFactory, TestCase, override_settings
 from django.test.client import Client
 from django.urls import resolve, reverse
 from django.utils.encoding import force_bytes
-
-import six
-import vcr
 from rosetta import views
 from rosetta.signals import entry_changed, post_save
 from rosetta.storage import get_storage
@@ -138,10 +136,10 @@ class RosettaTestCase(TestCase):
         self.copy_po_file_from_template('./django.po.issue67.template')
 
         # Make sure the plurals string is valid
-        with open(self.dest_file, 'rb') as f_:
+        with open(self.dest_file, 'r') as f_:
             content = f_.read()
-        self.assertTrue('Hello, world' not in six.text_type(content))
+        self.assertTrue('Hello, world' not in content)
-        self.assertTrue('|| n%100>=20) ? 1 : 2)' in six.text_type(content))
+        self.assertTrue('|| n%100>=20) ? 1 : 2)' in content)
         del content
 
         r = self.client.get(self.xx_form_url + '?msg_filter=untranslated')
@@ -157,7 +155,7 @@ class RosettaTestCase(TestCase):
         self.client.post(self.xx_form_url + '?msg_filter=untranslated', data)
 
         # Make sure the plurals string is still valid
-        with open(self.dest_file, 'rb') as f_:
+        with open(self.dest_file, 'r') as f_:
             content = f_.read()
         self.assertTrue('Hello, world' in str(content))
         self.assertTrue('|| n%100>=20) ? 1 : 2)' in str(content))
@@ -277,13 +275,13 @@ class RosettaTestCase(TestCase):
         # this user.
         with self.settings(ROSETTA_REQUIRES_AUTH=True):
             r = self.client3.get(self.xx_form_url)
-            self.assertFalse(r.content)
+            self.assertFalse(r.content.decode())
             self.assertEqual(r.status_code, 302)
 
         # When it's not required, we sail through.
         with self.settings(ROSETTA_REQUIRES_AUTH=False):
             r = self.client3.get(self.xx_form_url)
-            self.assertTrue(r.content)
+            self.assertTrue(r.content.decode())
             self.assertEqual(r.status_code, 200)
 
     @override_settings(ROSETTA_LANGUAGES=(('fr', 'French'), ('xx', 'Dummy Language')))
@@ -509,13 +507,13 @@ class RosettaTestCase(TestCase):
         data = {'m_e48f149a8b2e8baa81b816c0edf93890': 'Hello, world'}
         r = self.client.post(self.xx_form_url + '?filter=untranslated', data)
         # read the result
-        with open(self.dest_file, 'rb') as f_:
+        with open(self.dest_file, 'r') as f_:
-            content = six.text_type(f_.read())
+            content = f_.read()
 
         # make sure unicode data was properly converted to ascii
         self.assertTrue('Hello, world' in content)
         self.assertTrue('save_header_data@test.com' in content)
-        self.assertTrue('aeaeae aaaaaaa aaaa uuuu' in content)
+        self.assertTrue('aéaéaé aàaàaàa aâââ üüüü' in content)
 
     def test_24_percent_translation(self):
         self.copy_po_file_from_template('./django.po.template')
@@ -741,12 +739,8 @@ class RosettaTestCase(TestCase):
 
     def test_40_issue_155_auto_compile(self):
         def file_hash(file_string):
-            if six.PY3:
+            with open(file_string, encoding="latin-1") as file:
-                with open(file_string, encoding="latin-1") as file:
+                file_content = file.read().encode('utf-8')
-                    file_content = file.read().encode('utf-8')
-            else:
-                with open(file_string) as file:
-                    file_content = file.read()
             return hashlib.md5(file_content).hexdigest()
 
         def message_hashes():
@@ -886,7 +880,7 @@ class RosettaTestCase(TestCase):
 
         # But if the language isn't an option, we get a 404
         with self.settings(
-            ROSETTA_LANGUAGES=[l for l in settings.LANGUAGES if l[0] != 'xx']
+            ROSETTA_LANGUAGES=[lang for lang, __ in settings.LANGUAGES if lang != 'xx']
         ):
             view = self._setup_view(
                 view=views.TranslationFormView(), request=request, **kwargs

rosetta/views.py

@@ -2,20 +2,16 @@ import hashlib
 import os
 import os.path
 import re
-import unicodedata
 import zipfile
 from urllib.parse import urlencode
 
+import six
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import user_passes_test
 from django.core.paginator import Paginator
-from django.http import (
+from django.http import (Http404, HttpResponse, HttpResponseRedirect,
-    Http404,
+                         JsonResponse)
-    HttpResponse,
-    HttpResponseRedirect,
-    JsonResponse
-)
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.utils.encoding import force_bytes
@@ -23,8 +19,6 @@ from django.utils.functional import Promise, cached_property
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.cache import never_cache
 from django.views.generic import TemplateView, View
-
-import six
 from polib import pofile
 
 from . import get_version as get_rosetta_version
@@ -104,7 +98,7 @@ class RosettaFileLevelMixin(RosettaBaseMixin):
         """
         # (Formerly known as "rosetta_i18n_lang_code")
         lang_id = self.kwargs['lang_id']
-        if lang_id not in {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES}:
+        if lang_id not in {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES}:
             raise Http404
         if not can_translate_language(self.request.user, lang_id):
             raise Http404
@@ -227,7 +221,8 @@ class TranslationFileListView(RosettaBaseMixin, TemplateView):
                 third_party_apps=third_party_apps,
             )
             po_files = [
-                (get_app_name(l), os.path.realpath(l), pofile(l)) for l in po_paths
+                (get_app_name(lang), os.path.realpath(lang), pofile(lang))
+                for lang in po_paths
             ]
             po_files.sort(key=lambda app: app[0])
             languages.append((language[0], _(language[1]), po_files))
@@ -367,15 +362,11 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
 
         if file_change and self.po_file_is_writable:
             try:
-                self.po_file.metadata['Last-Translator'] = unicodedata.normalize(
+                self.po_file.metadata['Last-Translator'] = "{} {} <{}>".format(
-                    'NFKD',
+                    getattr(self.request.user, 'first_name', 'Anonymous'),
-                    u"%s %s <%s>"
+                    getattr(self.request.user, 'last_name', 'User'),
-                    % (
+                    getattr(self.request.user, 'email', 'anonymous@user.tld'),
-                        getattr(self.request.user, 'first_name', 'Anonymous'),
+                )
-                        getattr(self.request.user, 'last_name', 'User'),
-                        getattr(self.request.user, 'email', 'anonymous@user.tld'),
-                    ),
-                ).encode('ascii', 'ignore')
                 self.po_file.metadata['X-Translated-Using'] = u"django-rosetta %s" % (
                     get_rosetta_version()
                 )
@@ -412,7 +403,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
                         import uwsgi
 
                         uwsgi.reload()  # pretty easy right?
-                    except:
+                    except Exception:
                         pass  # we may not be running under uwsgi :P
                 # XXX: It would be nice to add a success message here!
             except Exception as e:
@@ -575,7 +566,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
         """
         ref_lang = self._request_request('ref_lang', 'msgid')
         if ref_lang != 'msgid':
-            allowed_languages = {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES}
+            allowed_languages = {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES}
             if ref_lang not in allowed_languages:
                 raise Http404
         return ref_lang

tox.ini

@@ -42,6 +42,7 @@ deps =
         goslate
         vcrpy
         coverage
+        pudb
 
 [testenv:gettext]
 basepython = python3
@@ -74,6 +75,6 @@ commands=
 
 [testenv:flake8]
 basepython = python3
-deps = flake8==2.4.1
+deps = flake8==3.9.2
 commands=
     flake8 {toxinidir}/rosetta