Find underlying type of related fields

We currently guess that all OneToOneFields are integers and use string
for everything else.

This is usually not an issue as Elasticsearch coerces between strings an
integers automatically. But this causes issues for the new facet feature
as Elasticsearch returns strings for ID fields that are actually
integers.

The field type changes shouldn't cause any trouble for existing indices
as Elasticsearch will continue to automatically coerce the types. Users
who want to use the new facet feature on related fields will have to
rebuild their index.

wagtail/search/backends/elasticsearch2.py

@@ -68,7 +68,6 @@ class Elasticsearch2Mapping:
         'IPAddressField': 'string',
         'GenericIPAddressField': 'string',
         'NullBooleanField': 'boolean',
-        'OneToOneField': 'integer',
         'PositiveIntegerField': 'integer',
         'PositiveSmallIntegerField': 'integer',
         'SlugField': 'string',

wagtail/search/index.py

@@ -201,7 +201,18 @@ class BaseField:
 
         try:
             field = self.get_field(cls)
+
+            # Follow foreign keys to find underlying type
+            # We use a while loop as it's possible for a foreign key
+            # to target a foreign key in another model.
+            # (for example, a foreign key to a child page model will
+            # point to the `page_ptr_id` field so we need to follow this
+            # second foreign key to find the `id`` field in the Page model)
+            while isinstance(field, RelatedField):
+                field = field.target_field
+
             return field.get_internal_type()
+
         except models.fields.FieldDoesNotExist:
             return 'CharField'
 

wagtail/search/tests/test_elasticsearch2_backend.py

@@ -531,6 +531,7 @@ class TestElasticsearch2Mapping(TestCase):
                             'date_of_birth_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -540,7 +541,7 @@ class TestElasticsearch2Mapping(TestCase):
                             'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}
+                    'tags_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -572,6 +573,7 @@ class TestElasticsearch2Mapping(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -612,13 +614,15 @@ class TestElasticsearch2MappingInheritance(TestCase):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'string', 'boost': 0.5, 'include_in_all': True}
+                            'name': {'type': 'string', 'boost': 0.5, 'include_in_all': True},
+                            'novel_id_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'string', 'boost': 0.25, 'include_in_all': True}
+                            'name': {'type': 'string', 'boost': 0.25, 'include_in_all': True},
                         }
                     },
 
@@ -636,6 +640,7 @@ class TestElasticsearch2MappingInheritance(TestCase):
                             'date_of_birth_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -645,7 +650,7 @@ class TestElasticsearch2MappingInheritance(TestCase):
                             'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}
+                    'tags_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -675,8 +680,10 @@ class TestElasticsearch2MappingInheritance(TestCase):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -704,6 +711,7 @@ class TestElasticsearch2MappingInheritance(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],

wagtail/search/tests/test_elasticsearch5_backend.py

@@ -532,6 +532,7 @@ class TestElasticsearch5Mapping(TestCase):
                             'date_of_birth_filter': {'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -541,7 +542,7 @@ class TestElasticsearch5Mapping(TestCase):
                             'slug_filter': {'type': 'keyword', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'type': 'keyword', 'include_in_all': False}
+                    'tags_filter': {'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -573,6 +574,7 @@ class TestElasticsearch5Mapping(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -613,9 +615,11 @@ class TestElasticsearch5MappingInheritance(TestCase):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'text', 'boost': 0.5, 'include_in_all': True}
+                            'name': {'type': 'text', 'boost': 0.5, 'include_in_all': True},
+                            'novel_id_filter': {'type': 'integer', 'include_in_all': False}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'type': 'integer', 'include_in_all': False},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
@@ -637,6 +641,7 @@ class TestElasticsearch5MappingInheritance(TestCase):
                             'date_of_birth_filter': {'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -646,7 +651,7 @@ class TestElasticsearch5MappingInheritance(TestCase):
                             'slug_filter': {'type': 'keyword', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'type': 'keyword', 'include_in_all': False}
+                    'tags_filter': {'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -676,8 +681,10 @@ class TestElasticsearch5MappingInheritance(TestCase):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -705,6 +712,7 @@ class TestElasticsearch5MappingInheritance(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],

wagtail/search/tests/test_elasticsearch6_backend.py

@@ -533,6 +533,7 @@ class TestElasticsearch6Mapping(TestCase):
                             'date_of_birth_filter': {'type': 'date'},
                         },
                     },
+                    'authors_filter': {'type': 'integer'},
                     'publication_date_filter': {'type': 'date'},
                     'number_of_pages_filter': {'type': 'integer'},
                     'tags': {
@@ -542,7 +543,7 @@ class TestElasticsearch6Mapping(TestCase):
                             'slug_filter': {'type': 'keyword'},
                         },
                     },
-                    'tags_filter': {'type': 'keyword'}
+                    'tags_filter': {'type': 'integer'}
                 }
             }
         }
@@ -574,6 +575,7 @@ class TestElasticsearch6Mapping(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -614,9 +616,11 @@ class TestElasticsearch6MappingInheritance(TestCase):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'text', 'boost': 0.5, 'copy_to': '_all_text'}
+                            'name': {'type': 'text', 'boost': 0.5, 'copy_to': '_all_text'},
+                            'novel_id_filter': {'type': 'integer'}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'type': 'integer'},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
@@ -639,6 +643,7 @@ class TestElasticsearch6MappingInheritance(TestCase):
                             'date_of_birth_filter': {'type': 'date'},
                         },
                     },
+                    'authors_filter': {'type': 'integer'},
                     'publication_date_filter': {'type': 'date'},
                     'number_of_pages_filter': {'type': 'integer'},
                     'tags': {
@@ -648,7 +653,7 @@ class TestElasticsearch6MappingInheritance(TestCase):
                             'slug_filter': {'type': 'keyword'},
                         },
                     },
-                    'tags_filter': {'type': 'keyword'}
+                    'tags_filter': {'type': 'integer'}
                 }
             }
         }
@@ -678,8 +683,10 @@ class TestElasticsearch6MappingInheritance(TestCase):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -707,6 +714,7 @@ class TestElasticsearch6MappingInheritance(TestCase):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],

wagtail/tests/search/models.py

@@ -28,6 +28,7 @@ class Book(index.Indexed, models.Model):
         index.SearchField('title', partial_match=True, boost=2.0),
         index.AutocompleteField('title'),
         index.FilterField('title'),
+        index.FilterField('authors'),
         index.RelatedFields('authors', Author.search_fields),
         index.FilterField('publication_date'),
         index.FilterField('number_of_pages'),
@@ -73,10 +74,6 @@ class Character(models.Model):
     name = models.CharField(max_length=255)
     novel = models.ForeignKey('Novel', related_name='characters', on_delete=models.CASCADE)
 
-    search_fields = [
-        index.SearchField('name'),
-    ]
-
     def __str__(self):
         return self.name
 
@@ -92,7 +89,9 @@ class Novel(Book):
         ]),
         index.RelatedFields('protagonist', [
             index.SearchField('name', boost=0.5),
+            index.FilterField('novel'),
         ]),
+        index.FilterField('protagonist'),
     ]