Rewrite spanned queries on all levels for defer/only (close #248).

Also:
- optimize fields to related model access
- more elegant way to access DefferedModel real class
- fix select_related rewriting and add some code to contemplate possible solutions
- update docs about select_related
- add 2 test for implemented features

docs/modeltranslation/usage.rst

@@ -125,6 +125,7 @@ These manager methods perform rewriting:
 - ``only()``, ``defer()``
 - ``values()``, ``values_list()``
 - ``dates()``
+- ``select_related()``
 - ``create()``, with optional auto-population_ feature
 
 In order not to introduce differences between ``X.objects.create(...)`` and ``X(...)``, model

modeltranslation/manager.py

@@ -5,6 +5,8 @@ django-linguo by Zach Mathew
 
 https://github.com/zmathew/django-linguo
 """
+import itertools
+
 from django.db import models
 from django.db.models import FieldDoesNotExist
 from django.db.models.fields.related import RelatedField, RelatedObject
@@ -46,14 +48,47 @@ def rewrite_lookup_key(model, lookup_key):
     if len(pieces) > 1:
         # Check if we are doing a lookup to a related trans model
         fields_to_trans_models = get_fields_to_translatable_models(model)
-        for field_to_trans, transmodel in fields_to_trans_models:
-            # Check ``original key``, as pieces[0] may have been already rewritten.
-            if original_key == field_to_trans:
-                pieces[1] = rewrite_lookup_key(transmodel, pieces[1])
-                break
+        # Check ``original key``, as pieces[0] may have been already rewritten.
+        if original_key in fields_to_trans_models:
+            transmodel = fields_to_trans_models[original_key]
+            pieces[1] = rewrite_lookup_key(transmodel, pieces[1])
     return '__'.join(pieces)
 
 
+def append_translated(model, fields):
+    "If translated field is encountered, add also all its translation fields."
+    fields = set(fields)
+    from modeltranslation.translator import translator
+    opts = translator.get_options_for_model(model)
+    for key, translated in opts.fields.items():
+        if key in fields:
+            fields = fields.union(f.name for f in translated)
+    return fields
+
+
+def append_lookup_key(model, lookup_key):
+    "Transform spanned__lookup__key into all possible translation versions, on all levels"
+    pieces = lookup_key.split('__', 1)
+
+    fields = append_translated(model, (pieces[0],))
+
+    if len(pieces) > 1:
+        # Check if we are doing a lookup to a related trans model
+        fields_to_trans_models = get_fields_to_translatable_models(model)
+        if pieces[0] in fields_to_trans_models:
+            transmodel = fields_to_trans_models[pieces[0]]
+            rest = append_lookup_key(transmodel, pieces[1])
+            fields = set('__'.join(pr) for pr in itertools.product(fields, rest))
+        else:
+            fields = set('%s__%s' % (f, pieces[1]) for f in fields)
+    return fields
+
+
+def append_lookup_keys(model, fields):
+    union = lambda x, y: x.union(y)
+    return reduce(union, (append_lookup_key(model, field) for field in fields), set())
+
+
 def rewrite_order_lookup_key(model, lookup_key):
     if lookup_key.startswith('-'):
         return '-' + rewrite_lookup_key(model, lookup_key[1:])
@@ -76,7 +111,7 @@ def get_fields_to_translatable_models(model):
             if isinstance(field_object, RelatedObject):
                 if get_translatable_fields_for_model(field_object.model) is not None:
                     results.append((field_name, field_object.model))
-        _F2TM_CACHE[model] = results
+        _F2TM_CACHE[model] = dict(results)
     return _F2TM_CACHE[model]
 
 _C2F_CACHE = {}
@@ -154,9 +189,13 @@ class MultilingualQuerySet(models.query.QuerySet):
     def select_related(self, *fields, **kwargs):
         if not self._rewrite:
             return super(MultilingualQuerySet, self).select_related(*fields, **kwargs)
+        # TO CONSIDER: whether this should rewrite only current language, or all languages?
+        # fk -> [fk, fk_en] (with en=active) VS fk -> [fk, fk_en, fk_de, fk_fr ...] (for all langs)
+
+        # new_args = append_lookup_keys(self.model, fields)
         new_args = []
         for key in fields:
-            new_args.append(rewrite_order_lookup_key(self.model, key))
+            new_args.append(rewrite_lookup_key(self.model, key))
         return super(MultilingualQuerySet, self).select_related(*new_args, **kwargs)
 
     # This method was not present in django-linguo
@@ -283,24 +322,14 @@ class MultilingualQuerySet(models.query.QuerySet):
         with auto_populate(self._populate_mode):
             return super(MultilingualQuerySet, self).get_or_create(**kwargs)
 
-    def _append_translated(self, fields):
-        "If translated field is encountered, add also all its translation fields."
-        fields = set(fields)
-        from modeltranslation.translator import translator
-        opts = translator.get_options_for_model(self.model)
-        for key, translated in opts.fields.items():
-            if key in fields:
-                fields = fields.union(f.name for f in translated)
-        return fields
-
     # This method was not present in django-linguo
     def defer(self, *fields):
-        fields = self._append_translated(fields)
+        fields = append_lookup_keys(self.model, fields)
         return super(MultilingualQuerySet, self).defer(*fields)
 
     # This method was not present in django-linguo
     def only(self, *fields):
-        fields = self._append_translated(fields)
+        fields = append_lookup_keys(self.model, fields)
         return super(MultilingualQuerySet, self).only(*fields)
 
     # This method was not present in django-linguo

modeltranslation/tests/models.py

@@ -64,6 +64,7 @@ class ForeignKeyModel(models.Model):
     optional = models.ForeignKey(TestModel, blank=True, null=True)
     hidden = models.ForeignKey(TestModel, blank=True, null=True, related_name="+")
     non = models.ForeignKey(NonTranslated, blank=True, null=True, related_name="test_fks")
+    untrans = models.ForeignKey(TestModel, blank=True, null=True, related_name="test_fks_un")
 
 
 class OneToOneFieldModel(models.Model):

modeltranslation/tests/tests.py

@@ -2737,6 +2737,39 @@ class TestManager(ModeltranslationTestBase):
             self.assertEqual('title_de', item.test.title)
             self.assertEqual('title_de', item.test.__class__.objects.only('title')[0].title)
 
+    def test_deferred_spanning(self):
+        test = models.TestModel.objects.create(title_de='title_de', title_en='title_en')
+        with auto_populate('all'):
+            models.ForeignKeyModel.objects.create(test=test)
+
+        item1 = models.ForeignKeyModel.objects.select_related("test").defer("test__text")[0].test
+        item2 = models.TestModel.objects.defer("text")[0]
+        self.assertIs(item1.__class__, item2.__class__)
+        # DeferredAttribute descriptors are present
+        self.assertIn('text_en', dir(item1.__class__))
+        self.assertIn('text_de', dir(item1.__class__))
+
+    def test_translation_fields_appending(self):
+        from modeltranslation.manager import append_lookup_keys, append_lookup_key
+        self.assertEqual(set(['untrans']), append_lookup_key(models.ForeignKeyModel, 'untrans'))
+        self.assertEqual(set(['title', 'title_en', 'title_de']),
+                         append_lookup_key(models.ForeignKeyModel, 'title'))
+        self.assertEqual(set(['test', 'test_en', 'test_de']),
+                         append_lookup_key(models.ForeignKeyModel, 'test'))
+        self.assertEqual(set(['title__eq', 'title_en__eq', 'title_de__eq']),
+                         append_lookup_key(models.ForeignKeyModel, 'title__eq'))
+        self.assertEqual(set(['test__smt', 'test_en__smt', 'test_de__smt']),
+                         append_lookup_key(models.ForeignKeyModel, 'test__smt'))
+        big_set = set(['test__url', 'test__url_en', 'test__url_de',
+                       'test_en__url', 'test_en__url_en', 'test_en__url_de',
+                       'test_de__url', 'test_de__url_en', 'test_de__url_de'])
+        self.assertEqual(big_set, append_lookup_key(models.ForeignKeyModel, 'test__url'))
+        self.assertEqual(set(['untrans__url', 'untrans__url_en', 'untrans__url_de']),
+                         append_lookup_key(models.ForeignKeyModel, 'untrans__url'))
+
+        self.assertEqual(big_set.union(['title', 'title_en', 'title_de']),
+                         append_lookup_keys(models.ForeignKeyModel, ['test__url', 'title']))
+
     def test_constructor_inheritance(self):
         inst = models.AbstractModelB()
         # Check if fields assigned in constructor hasn't been ignored.

modeltranslation/translator.py

@@ -487,7 +487,7 @@ class Translator(object):
         defined for the ``model`` and inherited from superclasses.
         """
         if model._deferred:
-            model = model.__bases__[0]
+            model = model._meta.proxy_for_model
         if model not in self._registry:
             # Create a new type for backwards compatibility.
             opts = type("%sTranslationOptions" % model.__name__,