Hopiu/django-rosetta
1
0
Fork 0
mirror of https://github.com/Hopiu/django-rosetta.git synced 2026-03-16 21:30:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixed unicode handling in gettext headers. Fixes #259

Browse source
This commit is contained in:
Marco Bonetti 2021-07-17 18:14:27 +02:00
parent 417094f0f1
commit cc12a30761
6 changed files with 52 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.pep8 → .flake8
View file

1
CHANGES
View file

@ -6,6 +6,7 @@ Version 0.9.7 (unreleased)
--------------------------
* Arabic translation. (#257, thanks @Bashar)
* Translations via the DeepL API (#258, thanks @halitcelik)
* Fixed unicode handling in gettext headers (#259, thanks @NotSqrt)
Version 0.9.6

30
rosetta/templatetags/rosetta.py
View file

@ -1,12 +1,12 @@
from django import template
from django.utils.safestring import mark_safe
from django.utils.html import escape
import re
import six
from django import template
from django.utils.html import escape
from django.utils.safestring import mark_safe
from rosetta.access import can_translate
register = template.Library()
rx = re.compile(r'(%(\([^\s\)]*\))?[sd]|\{[\w\d_]+?\})')
@ -14,33 +14,45 @@ can_translate = register.filter(can_translate)
def format_message(message):
return mark_safe(rx.sub('<code>\\1</code>', escape(message).replace(r'\n', '<br />\n')))
return mark_safe(
rx.sub('<code>\\1</code>', escape(message).replace(r'\n', '<br />\n'))
)
format_message = register.filter(format_message)
def lines_count(message):
return 1 + sum([len(line) / 50 for line in message.split('\n')])
lines_count = register.filter(lines_count)
def mult(a, b):
return int(a) * int(b)
mult = register.filter(mult)
def minus(a, b):
try:
return int(a) - int(b)
except:
except Exception:
return 0
minus = register.filter(minus)
def gt(a, b):
try:
return int(a) > int(b)
except:
except Exception:
return False
gt = register.filter(gt)
@ -54,6 +66,8 @@ def do_incr(parser, token):
if name not in parser._namedIncrNodes:
parser._namedIncrNodes[name] = IncrNode(0)
return parser._namedIncrNodes[name]
do_incr = register.tag('increment', do_incr)
@ -68,4 +82,6 @@ class IncrNode(template.Node):
def is_fuzzy(message):
return message and hasattr(message, 'flags') and 'fuzzy' in message.flags
is_fuzzy = register.filter(is_fuzzy)

32
rosetta/tests/tests.py
View file

@ -4,6 +4,7 @@ import os
import shutil
from urllib.parse import urlencode
import vcr
from django import VERSION
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
@ -13,9 +14,6 @@ from django.test import RequestFactory, TestCase, override_settings
from django.test.client import Client
from django.urls import resolve, reverse
from django.utils.encoding import force_bytes
import six
import vcr
from rosetta import views
from rosetta.signals import entry_changed, post_save
from rosetta.storage import get_storage
@ -138,10 +136,10 @@ class RosettaTestCase(TestCase):
self.copy_po_file_from_template('./django.po.issue67.template')
# Make sure the plurals string is valid
with open(self.dest_file, 'rb') as f_:
with open(self.dest_file, 'r') as f_:
content = f_.read()
self.assertTrue('Hello, world' not in six.text_type(content))
self.assertTrue('|| n%100>=20) ? 1 : 2)' in six.text_type(content))
self.assertTrue('Hello, world' not in content)
self.assertTrue('|| n%100>=20) ? 1 : 2)' in content)
del content
r = self.client.get(self.xx_form_url + '?msg_filter=untranslated')
@ -157,7 +155,7 @@ class RosettaTestCase(TestCase):
self.client.post(self.xx_form_url + '?msg_filter=untranslated', data)
# Make sure the plurals string is still valid
with open(self.dest_file, 'rb') as f_:
with open(self.dest_file, 'r') as f_:
content = f_.read()
self.assertTrue('Hello, world' in str(content))
self.assertTrue('|| n%100>=20) ? 1 : 2)' in str(content))
@ -277,13 +275,13 @@ class RosettaTestCase(TestCase):
# this user.
with self.settings(ROSETTA_REQUIRES_AUTH=True):
r = self.client3.get(self.xx_form_url)
self.assertFalse(r.content)
self.assertFalse(r.content.decode())
self.assertEqual(r.status_code, 302)
# When it's not required, we sail through.
with self.settings(ROSETTA_REQUIRES_AUTH=False):
r = self.client3.get(self.xx_form_url)
self.assertTrue(r.content)
self.assertTrue(r.content.decode())
self.assertEqual(r.status_code, 200)
@override_settings(ROSETTA_LANGUAGES=(('fr', 'French'), ('xx', 'Dummy Language')))
@ -509,13 +507,13 @@ class RosettaTestCase(TestCase):
data = {'m_e48f149a8b2e8baa81b816c0edf93890': 'Hello, world'}
r = self.client.post(self.xx_form_url + '?filter=untranslated', data)
# read the result
with open(self.dest_file, 'rb') as f_:
content = six.text_type(f_.read())
with open(self.dest_file, 'r') as f_:
content = f_.read()
# make sure unicode data was properly converted to ascii
self.assertTrue('Hello, world' in content)
self.assertTrue('save_header_data@test.com' in content)
self.assertTrue('aeaeae aaaaaaa aaaa uuuu' in content)
self.assertTrue('aéaéaé aàaàaàa aâââ üüüü' in content)
def test_24_percent_translation(self):
self.copy_po_file_from_template('./django.po.template')
@ -741,12 +739,8 @@ class RosettaTestCase(TestCase):
def test_40_issue_155_auto_compile(self):
def file_hash(file_string):
if six.PY3:
with open(file_string, encoding="latin-1") as file:
file_content = file.read().encode('utf-8')
else:
with open(file_string) as file:
file_content = file.read()
with open(file_string, encoding="latin-1") as file:
file_content = file.read().encode('utf-8')
return hashlib.md5(file_content).hexdigest()
def message_hashes():
@ -886,7 +880,7 @@ class RosettaTestCase(TestCase):
# But if the language isn't an option, we get a 404
with self.settings(
ROSETTA_LANGUAGES=[l for l in settings.LANGUAGES if l[0] != 'xx']
ROSETTA_LANGUAGES=[lang for lang, __ in settings.LANGUAGES if lang != 'xx']
):
view = self._setup_view(
view=views.TranslationFormView(), request=request, **kwargs

35
rosetta/views.py
View file

@ -2,20 +2,16 @@ import hashlib
import os
import os.path
import re
import unicodedata
import zipfile
from urllib.parse import urlencode
import six
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.paginator import Paginator
from django.http import (
Http404,
HttpResponse,
HttpResponseRedirect,
JsonResponse
)
from django.http import (Http404, HttpResponse, HttpResponseRedirect,
JsonResponse)
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.utils.encoding import force_bytes
@ -23,8 +19,6 @@ from django.utils.functional import Promise, cached_property
from django.utils.translation import gettext_lazy as _
from django.views.decorators.cache import never_cache
from django.views.generic import TemplateView, View
import six
from polib import pofile
from . import get_version as get_rosetta_version
@ -104,7 +98,7 @@ class RosettaFileLevelMixin(RosettaBaseMixin):
"""
# (Formerly known as "rosetta_i18n_lang_code")
lang_id = self.kwargs['lang_id']
if lang_id not in {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES}:
if lang_id not in {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES}:
raise Http404
if not can_translate_language(self.request.user, lang_id):
raise Http404
@ -227,7 +221,8 @@ class TranslationFileListView(RosettaBaseMixin, TemplateView):
third_party_apps=third_party_apps,
)
po_files = [
(get_app_name(l), os.path.realpath(l), pofile(l)) for l in po_paths
(get_app_name(lang), os.path.realpath(lang), pofile(lang))
for lang in po_paths
]
po_files.sort(key=lambda app: app[0])
languages.append((language[0], _(language[1]), po_files))
@ -367,15 +362,11 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
if file_change and self.po_file_is_writable:
try:
self.po_file.metadata['Last-Translator'] = unicodedata.normalize(
'NFKD',
u"%s %s <%s>"
% (
getattr(self.request.user, 'first_name', 'Anonymous'),
getattr(self.request.user, 'last_name', 'User'),
getattr(self.request.user, 'email', 'anonymous@user.tld'),
),
).encode('ascii', 'ignore')
self.po_file.metadata['Last-Translator'] = "{} {} <{}>".format(
getattr(self.request.user, 'first_name', 'Anonymous'),
getattr(self.request.user, 'last_name', 'User'),
getattr(self.request.user, 'email', 'anonymous@user.tld'),
)
self.po_file.metadata['X-Translated-Using'] = u"django-rosetta %s" % (
get_rosetta_version()
)
@ -412,7 +403,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
import uwsgi
uwsgi.reload() # pretty easy right?
except:
except Exception:
pass # we may not be running under uwsgi :P
# XXX: It would be nice to add a success message here!
except Exception as e:
@ -575,7 +566,7 @@ class TranslationFormView(RosettaFileLevelMixin, TemplateView):
"""
ref_lang = self._request_request('ref_lang', 'msgid')
if ref_lang != 'msgid':
allowed_languages = {l[0] for l in rosetta_settings.ROSETTA_LANGUAGES}
allowed_languages = {lang[0] for lang in rosetta_settings.ROSETTA_LANGUAGES}
if ref_lang not in allowed_languages:
raise Http404
return ref_lang

3
tox.ini
View file

@ -42,6 +42,7 @@ deps =
goslate
vcrpy
coverage
pudb
[testenv:gettext]
basepython = python3
@ -74,6 +75,6 @@ commands=
[testenv:flake8]
basepython = python3
deps = flake8==2.4.1
deps = flake8==3.9.2
commands=
flake8 {toxinidir}/rosetta