From d6f59672ead17bc21d19d4f3f9a5658615dd26ec Mon Sep 17 00:00:00 2001 From: David McCann Date: Tue, 14 Sep 2010 10:10:20 +0300 Subject: [PATCH 1/5] refactored expand_filter_string from recursive to iterative --- managers.py | 98 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/managers.py b/managers.py index 987a7d8..5776cc2 100644 --- a/managers.py +++ b/managers.py @@ -2,51 +2,75 @@ from django.db import models #from django.db.models import Aggregate from .models import EavAttribute - -def expand_filter_string(model_cls, q_str, prefix='', extra_filters=None): +def expand_filter_string(q_str, root_cls): from .utils import EavRegistry - if not extra_filters: - extra_filters = {} - fields = q_str.split('__') + extra_filters = {} + # q_str is a filter argument, something like: + # zoo__animal__food__eav__vitamin_c__gt, where + # in this example 'food' would be registered as an entity, + # 'eav' is the proxy field name, and 'contains_vitamin_c' is the + # attribute. So once we split the above example would be a list with + # something we can easily process: + # ['zoo','animal','food','eav','vitamin_c', 'gt'] + filter_tokens = q_str.split('__') + current_cls = root_cls + upperbound = len(filter_tokens) - 1 + i = 0 + while i < upperbound: + current_cls_config = EavRegistry.get_config_cls_for_model(current_cls) + if current_cls_config and filter_tokens[i] == current_cls_config.proxy_field_name: + gr_field = current_cls_config.generic_relation_field_name + # this will always work, because we're iterating over the length of the tokens - 1. + # if someone just specifies 'zoo__animal_food__eav' as a filter, + # they'll get the appropriate error: column 'eav' doesn't existt (i.e. if i != 0), + # prepend all the + slug = filter_tokens[i + 1] + datatype = EavAttribute.objects.get(slug=slug).datatype + extra_filter_key = '%s__attribute__slug' % gr_field + # if we're somewhere in the middle of this filter argument + # joins up to this point + if i != 0: + extra_filter_key = '%s__%s' % ('__'.join(filter_tokens[0:i]), extra_filter_key) + extra_filters[extra_filter_key] = slug + # modify the filter argument in-place, expanding 'eav' into 'eav_values__value_' + filter_tokens = filter_tokens[0:i] + [gr_field, 'value_%s' % datatype] + filter_tokens[i + 2:] + # this involves a little indexing voodoo, because we inserted two elements in place of one + # original element + i += 1 + upperbound += 1 +# filter_tokens[0] = "%s__value_%s" % (gr_field, datatype) + else: + direct = False + # Is it not EAV, but also not another field? + try: + field_object, model, direct, m2m = current_cls._meta.get_field_by_name(filter_tokens[0]) + # If we've hit the end, i.e. a simple column attribute like IntegerField or Boolean Field, + # we're done modifying the tokens, so we can break out of the loop early + if direct: + return '__'.join(filter_tokens), extra_filters + else: + # It is a foreign key to some other model, so we need to keep iterating, looking for registered + # entity models to expand + current_cls = field_object.model + except models.FieldDoesNotExist: + # this is a bogus filter anyway on a non-existent attribute, let the call to the super filter throw the + # appropriate error + return '__'.join(filter_tokens), extra_filters - # Is it EAV? - config_cls = EavRegistry.get_config_cls_for_model(model_cls) - if len(fields) > 1 and config_cls and fields[0] == config_cls.proxy_field_name: - gr_field = config_cls.generic_relation_field_name - slug = fields[1] - datatype = EavAttribute.objects.get(slug=slug).datatype - extra_filter_key = '%s__attribute__slug' % gr_field - if prefix: - extra_filter_key = '%s__%s' % (prefix, extra_filter_key) - extra_filters[extra_filter_key] = slug - fields[0] = "%s__value_%s" % (gr_field, datatype) - fields.pop(1) - return '__'.join(fields), extra_filters - - - direct = False - # Is it not EAV, but also not another field? - try: - field_object, model, direct, m2m = model_cls._meta.get_field_by_name(fields[0]) - except models.FieldDoesNotExist: - return q_str, extra_filters - - # Is it a direct field? - if direct: - return q_str, extra_filters - else: - # It is a foreign key. - prefix = "%s__%s" % (prefix, fields[0]) if prefix else fields[0] - sub_q_str = '__'.join(fields[1:]) - retstring, dictionary = self.expand_filter_string(field_object.model, sub_q_str, prefix, extra_filters) - return ("%s__%s" % (fields[0], retstring), dictionary) + # regular loop forward + i += 1 + # at the end of the day, we return the modified keyword filter, and any additional filters needed to make this + # query work, for passing up to the super call to filter() + return '__'.join(filter_tokens), extra_filters class EntityManager(models.Manager): def filter(self, *args, **kwargs): qs = self.get_query_set().filter(*args) cls = self.model for lookup, value in kwargs.items(): - updated_lookup, extra_filters = expand_filter_string(cls, lookup) + print "In %s" % lookup + updated_lookup, extra_filters = expand_filter_string(lookup, cls) + print "Out %s %s" % (updated_lookup, extra_filters) extra_filters.update({updated_lookup: value}) qs = qs.filter(**extra_filters) return qs From 80bcb8cdf204c11474b82854d6a035496850d03d Mon Sep 17 00:00:00 2001 From: David McCann Date: Tue, 14 Sep 2010 10:55:48 +0300 Subject: [PATCH 2/5] removed debugging code --- managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/managers.py b/managers.py index 5776cc2..868f570 100644 --- a/managers.py +++ b/managers.py @@ -68,9 +68,7 @@ class EntityManager(models.Manager): qs = self.get_query_set().filter(*args) cls = self.model for lookup, value in kwargs.items(): - print "In %s" % lookup updated_lookup, extra_filters = expand_filter_string(lookup, cls) - print "Out %s %s" % (updated_lookup, extra_filters) extra_filters.update({updated_lookup: value}) qs = qs.filter(**extra_filters) return qs From 9ae619a2258ffa4ccf408cb9fa8b494bff3ea071 Mon Sep 17 00:00:00 2001 From: David McCann Date: Tue, 14 Sep 2010 10:56:19 +0300 Subject: [PATCH 3/5] added manager_only optional argument (for registering non-entity Models, to allow them to do fancy filtering) --- utils.py | 81 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/utils.py b/utils.py index d70632a..1dd9eb5 100644 --- a/utils.py +++ b/utils.py @@ -76,7 +76,7 @@ class EavRegistry(object): @staticmethod - def register(model_cls, config_cls=EavConfig): + def register(model_cls, config_cls=EavConfig, manager_only=False): """ Inject eav features into the given model and attach a signal listener to it for setup. @@ -89,14 +89,16 @@ class EavRegistry(object): config_cls = EavRegistry.wrap_config_class(model_cls, config_cls) - # we want to call attach and save handler on instance creation and - # saving - post_init.connect(EavRegistry.attach, sender=model_cls) - post_save.connect(EavEntity.save_handler, sender=model_cls) + if not manager_only: + # we want to call attach and save handler on instance creation and + # saving + post_init.connect(EavRegistry.attach, sender=model_cls) + post_save.connect(EavEntity.save_handler, sender=model_cls) # todo: rename cache in data EavRegistry.cache[cls_id] = { 'config_cls': config_cls, - 'model_cls': model_cls } + 'model_cls': model_cls, + 'manager_only': manager_only } # save the old manager if the attribute name conflict with the new # one @@ -104,43 +106,44 @@ class EavRegistry(object): mgr = getattr(model_cls, config_cls.manager_field_name) EavRegistry.cache[cls_id]['old_mgr'] = mgr - # set add the config_cls as an attribute of the model - # it will allow to perform some operation directly from this model - setattr(model_cls, config_cls.proxy_field_name, config_cls) - - # todo : not useful anymore ? - setattr(getattr(model_cls, config_cls.proxy_field_name), - 'get_eav_attributes', config_cls.get_eav_attributes) + if not manager_only: + # set add the config_cls as an attribute of the model + # it will allow to perform some operation directly from this model + setattr(model_cls, config_cls.proxy_field_name, config_cls) + + # todo : not useful anymore ? + setattr(getattr(model_cls, config_cls.proxy_field_name), + 'get_eav_attributes', config_cls.get_eav_attributes) # attache the new manager to the model mgr = EntityManager() mgr.contribute_to_class(model_cls, config_cls.manager_field_name) - # todo: see with david how to change that - try: - EavEntity.update_attr_cache_for_model(model_cls) - except DatabaseError: - pass - - # todo: make that overridable - # attach the generic relation to the model - if config_cls.generic_relation_field_related_name: - rel_name = config_cls.generic_relation_field_related_name - else: - rel_name = model_cls.__name__ - gr_name = config_cls.generic_relation_field_name.lower() - generic_relation = generic.GenericRelation(EavValue, - object_id_field='entity_id', - content_type_field='entity_ct', - related_name=rel_name) - generic_relation.contribute_to_class(model_cls, gr_name) + if not manager_only: + # todo: see with david how to change that + try: + EavEntity.update_attr_cache_for_model(model_cls) + except DatabaseError: + pass + # todo: make that overridable + # attach the generic relation to the model + if config_cls.generic_relation_field_related_name: + rel_name = config_cls.generic_relation_field_related_name + else: + rel_name = model_cls.__name__ + gr_name = config_cls.generic_relation_field_name.lower() + generic_relation = generic.GenericRelation(EavValue, + object_id_field='entity_id', + content_type_field='entity_ct', + related_name=rel_name) + generic_relation.contribute_to_class(model_cls, gr_name) @staticmethod def unregister(model_cls): """ - Inject eav features into the given model and attach a signal - listener to it for setup. + Do the INVERSE of 'Inject eav features into the given model + and attach a signal listener to it for setup.' """ cls_id = get_unique_class_identifier(model_cls) @@ -149,8 +152,10 @@ class EavRegistry(object): cache = EavRegistry.cache[cls_id] config_cls = cache['config_cls'] - post_init.disconnect(EavRegistry.attach, sender=model_cls) - post_save.disconnect(EavEntity.save_handler, sender=model_cls) + manager_only = cache['manager_only'] + if not manager_only: + post_init.disconnect(EavRegistry.attach, sender=model_cls) + post_save.disconnect(EavEntity.save_handler, sender=model_cls) try: delattr(model_cls, config_cls.manager_field_name) @@ -168,7 +173,6 @@ class EavRegistry(object): except AttributeError: pass - if 'old_mgr' in cache: cache['old_mgr'].contribute_to_class(model_cls, config_cls.manager_field_name) @@ -178,10 +182,11 @@ class EavRegistry(object): except AttributeError: pass - EavEntity.flush_attr_cache_for_model(model_cls) + if not manager_only: + EavEntity.flush_attr_cache_for_model(model_cls) + EavRegistry.cache.pop(cls_id) - # todo : test cache # todo : tst unique identitfier # todo: test update attribute cache on attribute creation From cda3bbab2b24f565d38d23ff66dee045bc57387a Mon Sep 17 00:00:00 2001 From: David McCann Date: Tue, 14 Sep 2010 12:50:27 +0300 Subject: [PATCH 4/5] added Q filtering magic to the eav manager. Recursively rebuilds the Q tree in-place before passing to the super class' filter. --- managers.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/managers.py b/managers.py index 868f570..1e739e6 100644 --- a/managers.py +++ b/managers.py @@ -1,4 +1,5 @@ from django.db import models +from django.db.models import Q #from django.db.models import Aggregate from .models import EavAttribute @@ -63,10 +64,38 @@ def expand_filter_string(q_str, root_cls): # query work, for passing up to the super call to filter() return '__'.join(filter_tokens), extra_filters +def expand_q_filters(q, root_cls): + new_children = [] + for qi in q.children: + if type(qi) is tuple: + # this child is a leaf node: in Q this is a 2-tuple of: + # (filter parameter, value) + expanded_string, extra_filters = expand_filter_string(qi[0], root_cls) + extra_filters.update({expanded_string: qi[1]}) + if q.connector == 'OR': + # if it's an or, we now have additional filters that need + # to be ANDed together, so we have to make a sub-Q child + # in place of the original tuple + new_children.append(Q(**extra_filters)) + else: + # otherwise, we can just append all the new filters, they're + # ANDed together anyway + for k,v in extra_filters.items(): + new_children.append((k,v)) + else: + # this child is another Q node: recursify! + new_children.append(expand_q_filters(qi, root_cls)) + q.children = new_children + return q + class EntityManager(models.Manager): def filter(self, *args, **kwargs): - qs = self.get_query_set().filter(*args) cls = self.model + for arg in args: + if isinstance(arg, Q): + # modify Q objects in-place (warning: recursion ahead) + expand_q_filters(arg, cls) + qs = self.get_query_set().filter(*args) for lookup, value in kwargs.items(): updated_lookup, extra_filters = expand_filter_string(lookup, cls) extra_filters.update({updated_lookup: value}) From 267c41f06d9c19c5a73d4e0ae7aa3cf9e5a9fbd9 Mon Sep 17 00:00:00 2001 From: David McCann Date: Tue, 14 Sep 2010 12:57:26 +0300 Subject: [PATCH 5/5] add Q modification to exclude() as well as filter --- managers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/managers.py b/managers.py index 1e739e6..5887c16 100644 --- a/managers.py +++ b/managers.py @@ -103,8 +103,12 @@ class EntityManager(models.Manager): return qs def exclude(self, *args, **kwargs): - qs = self.get_query_set().exclude(*args) cls = self.model + for arg in args: + if isinstance(arg, Q): + # modify Q objects in-place (warning: recursion ahead) + expand_q_filters(arg, cls) + qs = self.get_query_set().exclude(*args) for lookup, value in kwargs.items(): lookups = self._filter_by_lookup(qs, lookup, value) updated_lookup, extra_filters = expand_filter_string(cls, lookup)