Adding CSV datatype + CSVField (widget, formfield also) + validators + tests

eav/fields.py

@@ -1,9 +1,11 @@
 import re
 
-from django.core.exceptions import ValidationError
 from django.db import models
+from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 
+from .forms import CSVFormField
+
 class EavSlugField(models.SlugField):
     """
     The slug field used by :class:`~eav.models.Attribute`
@@ -60,3 +62,48 @@ class EavDatatypeField(models.CharField):
             raise ValidationError(_(
                 'You cannot change the datatype of an attribute that is already in use.'
             ))
+
+
+class CSVField(models.TextField): # (models.Field):
+    description = _("A Comma-Separated-Value field.")
+    default_separator = ";"
+
+    def __init__(self, separator=";", *args, **kwargs):
+        self.separator = separator
+        kwargs.setdefault('default', "")
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if self.separator != self.default_separator:
+            kwargs['separator'] = self.separator
+        return name, path, args, kwargs
+
+    def formfield(self, **kwargs):
+        defaults = {'form_class': CSVFormField}
+        defaults.update(kwargs)
+        return super().formfield(**defaults)
+
+    def from_db_value(self, value, expression, connection, context=None):
+        if value is None:
+            return []
+        return value.split(self.separator)
+
+    def to_python(self, value):
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return value
+        return value.split(self.separator)
+
+    def get_prep_value(self, value):
+        if not value:
+            return ""
+        if isinstance(value, str):
+            return value
+        elif isinstance(value, list):
+            return self.separator.join(value)
+
+    def value_to_string(self, obj):
+        value = self.value_from_object(obj)
+        return self.get_prep_value(value)

eav/forms.py

@@ -2,17 +2,39 @@
 
 from copy import deepcopy
 
+from django import forms
 from django.contrib.admin.widgets import AdminSplitDateTime
 from django.forms import (BooleanField, CharField, ChoiceField, DateTimeField,
                           FloatField, IntegerField, ModelForm)
+from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 
-
 try:
     from django.forms import JSONField
 except:
     JSONField = CharField
 
+from .widgets import CSVWidget
+
+
+class CSVFormField(forms.Field):
+    message = _('Enter comma-separated-values. eg: one;two;three.')
+    code = 'invalid'
+    widget = CSVWidget
+    default_separator = ";"
+
+    def to_python(self, value):
+        if not value:
+            return []
+        return [v.strip() for v in value.split(self.separator) if v]
+
+    def validate(self, value):
+        super().validate(value)
+        try:
+            isinstance(value.split(self.separator), list)
+        except ValidationError:
+            raise ValidationError(self.message, code=self.code)
+
 
 class BaseDynamicEntityForm(ModelForm):
     """
@@ -35,16 +57,18 @@ class BaseDynamicEntityForm(ModelForm):
     bool   BooleanField
     enum   ChoiceField
     json   JSONField
+    csv    CSVField
     =====  =============
     """
     FIELD_CLASSES = {
-        'text': CharField,
+        'text':  CharField,
         'float': FloatField,
-        'int': IntegerField,
-        'date': DateTimeField,
-        'bool': BooleanField,
-        'enum': ChoiceField,
-        'json': JSONField,
+        'int':   IntegerField,
+        'date':  DateTimeField,
+        'bool':  BooleanField,
+        'enum':  ChoiceField,
+        'json':  JSONField,
+        'csv':   CSVFormField,
     }
 
     def __init__(self, data=None, *args, **kwargs):

eav/migrations/0005_auto_20210510_1305.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2 on 2021-05-10 13:05
+
+from django.db import migrations
+import eav.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('eav', '0004_alter_value_value_bool'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='value',
+            name='value_csv',
+            field=eav.fields.CSVField(blank=True, default=[], null=True),
+        ),
+        migrations.AlterField(
+            model_name='attribute',
+            name='datatype',
+            field=eav.fields.EavDatatypeField(choices=[('text', 'Text'), ('date', 'Date'), ('float', 'Float'), ('int', 'Integer'), ('bool', 'True / False'), ('object', 'Django Object'), ('enum', 'Multiple Choice'), ('json', 'JSON Object'), ('csv', 'Comma-Separated-Value')], max_length=6, verbose_name='Data Type'),
+        ),
+    ]

eav/models.py

@@ -37,10 +37,11 @@ from .validators import (
     validate_bool,
     validate_object,
     validate_enum,
-    validate_json
+    validate_json,
+    validate_csv,
 )
 from .exceptions import IllegalAssignmentException
-from .fields import EavDatatypeField, EavSlugField
+from .fields import EavDatatypeField, EavSlugField, CSVField
 from . import register
 
 
@@ -119,6 +120,8 @@ class Attribute(models.Model):
         * object (TYPE_OBJECT)
         * enum (TYPE_ENUM)
         * json (TYPE_JSON)
+        * csv (TYPE_CSV)
+
 
     Examples::
 
@@ -151,6 +154,7 @@ class Attribute(models.Model):
     TYPE_OBJECT  = 'object'
     TYPE_ENUM    = 'enum'
     TYPE_JSON    = 'json'
+    TYPE_CSV     = 'csv'
 
     DATATYPE_CHOICES = (
         (TYPE_TEXT,    _('Text')),
@@ -161,6 +165,7 @@ class Attribute(models.Model):
         (TYPE_OBJECT,  _('Django Object')),
         (TYPE_ENUM,    _('Multiple Choice')),
         (TYPE_JSON,    _('JSON Object')),
+        (TYPE_CSV,     _('Comma-Separated-Value')),
     )
 
     # Core attributes
@@ -263,6 +268,7 @@ class Attribute(models.Model):
             'object': validate_object,
             'enum':   validate_enum,
             'json':   validate_json,
+            'csv':    validate_csv,
         }
 
         return [DATATYPE_VALIDATORS[self.datatype]]
@@ -398,6 +404,7 @@ class Value(models.Model):
     value_date  = models.DateTimeField(blank = True, null = True)
     value_bool  = models.BooleanField(blank = True, null = True)
     value_json  = JSONField(default=dict, encoder=DjangoJSONEncoder, blank = True, null = True)
+    value_csv   = CSVField(blank = True, null = True)
 
     value_enum  = models.ForeignKey(
         EnumValue,
@@ -555,7 +562,7 @@ class Entity(object):
             if self._hasattr(attribute.slug):
                 attribute_value = self._getattr(attribute.slug)
                 if attribute.datatype == Attribute.TYPE_ENUM and not isinstance(attribute_value, EnumValue):
-                    if attribute_value is not None: 
+                    if attribute_value is not None:
                         attribute_value = EnumValue.objects.get(value=attribute_value)
                 attribute.save_value(self.instance, attribute_value)
 

eav/validators.py

@@ -97,3 +97,13 @@ def validate_json(value):
             raise ValidationError(_(u"Must be a JSON Serializable object"))
     except ValueError:
         raise ValidationError(_(u"Must be a JSON Serializable object"))
+
+
+def validate_csv(value):
+    """
+    Raises ``ValidationError`` unless *value* is a c-s-v value.
+    """
+    if isinstance(value, str):
+        value = value.split(";")
+    if not isinstance(value, list):
+        raise ValidationError(_(u"Must be Comma-Separated-Value."))

eav/widgets.py

@@ -0,0 +1,23 @@
+from django.forms.widgets import Textarea
+from django.core.exceptions import ValidationError
+from django.core import validators
+
+EMPTY_VALUES = validators.EMPTY_VALUES + ('[]', )
+
+class CSVWidget(Textarea):
+    is_hidden = False
+
+    def prep_value(self, value):
+        """ Prepare value before effectively render widget """
+        if value in EMPTY_VALUES:
+            return ""
+        elif isinstance(value, str):
+            return value
+        elif isinstance(value, list):
+            return ";".join(value)
+        raise ValidationError('Invalid format.')
+
+    def render(self, name, value, **kwargs):
+        value = self.prep_value(value)
+        return super().render(name, value, **kwargs)
+

tests/data_validation.py

@@ -22,6 +22,7 @@ class DataValidation(TestCase):
         Attribute.objects.create(name='Pregnant?', datatype=Attribute.TYPE_BOOLEAN)
         Attribute.objects.create(name='User', datatype=Attribute.TYPE_OBJECT)
         Attribute.objects.create(name='Extra', datatype=Attribute.TYPE_JSON)
+        Attribute.objects.create(name='Multi', datatype=Attribute.TYPE_CSV)
 
     def tearDown(self):
         eav.unregister(Patient)
@@ -195,3 +196,12 @@ class DataValidation(TestCase):
         p.eav.extra = {"eyes": "blue", "hair": "brown"}
         p.save()
         self.assertEqual(Patient.objects.get(pk=p.pk).eav.extra.get("eyes", ""), "blue")
+
+    def test_csv_validation(self):
+        yes = EnumValue.objects.create(value='yes')
+        p = Patient.objects.create(name='Mike')
+        p.eav.multi = yes
+        self.assertRaises(ValidationError, p.save)
+        p.eav.multi = "one;two;three"
+        p.save()
+        self.assertEqual(Patient.objects.get(pk=p.pk).eav.multi, ["one","two","three"])

tests/queries.py

@@ -21,6 +21,7 @@ class Queries(TestCase):
         Attribute.objects.create(name='city', datatype=Attribute.TYPE_TEXT)
         Attribute.objects.create(name='country', datatype=Attribute.TYPE_TEXT)
         Attribute.objects.create(name='extras', datatype=Attribute.TYPE_JSON)
+        Attribute.objects.create(name='illness', datatype=Attribute.TYPE_CSV)
 
         self.yes = EnumValue.objects.create(value='yes')
         self.no = EnumValue.objects.create(value='no')
@@ -42,12 +43,29 @@ class Queries(TestCase):
         no = self.no
 
         data = [
-            # Name,    age, fever, city,       country,   extras
-            ['Anne',   3,   no,    'New York', 'USA',     {"chills": "yes"}],
-            ['Bob',    15,  no,    'Bamako',   'Mali',    {}],
-            ['Cyrill', 15,  yes,   'Kisumu',   'Kenya',   {"chills": "yes", "headache": "no"}],
-            ['Daniel', 3,   no,    'Nice',     'France',  {"headache": "yes"}],
-            ['Eugene', 2,   yes,   'France',   'Nice',    {"chills": "no", "headache": "yes"}]
+            # Name, age, fever,
+            # city, country, extras
+            # possible illness
+            ['Anne',   3,   no,
+             'New York', 'USA',     {"chills": "yes"},
+             "cold"
+             ],
+            ['Bob',    15,  no,
+             'Bamako',   'Mali',    {},
+             ""
+             ],
+            ['Cyrill', 15,  yes,
+             'Kisumu',   'Kenya',   {"chills": "yes", "headache": "no"},
+             "flu"
+             ],
+            ['Daniel', 3,   no,
+             'Nice',     'France',  {"headache": "yes"},
+             "cold"
+             ],
+            ['Eugene', 2,   yes,
+             'France',   'Nice',    {"chills": "no", "headache": "yes"},
+             "flu;cold"
+             ]
         ]
 
         for row in data:
@@ -57,7 +75,8 @@ class Queries(TestCase):
                 eav__fever=row[2],
                 eav__city=row[3],
                 eav__country=row[4],
-                eav__extras=row[5]
+                eav__extras=row[5],
+                eav__illness=row[6]
             )
 
     def test_get_or_create_with_eav(self):
@@ -83,7 +102,7 @@ class Queries(TestCase):
 
         # Check number of objects in DB.
         self.assertEqual(Patient.objects.count(), 5)
-        self.assertEqual(Value.objects.count(), 25)
+        self.assertEqual(Value.objects.count(), 29)
 
         # Nobody
         q1 = Q(eav__fever=self.yes) & Q(eav__fever=self.no)
@@ -183,6 +202,23 @@ class Queries(TestCase):
         p = Patient.objects.exclude(q1)
         self.assertEqual(p.count(), 4)
 
+        # Illness:
+        # Cold
+        q1 = Q(eav__illness__icontains="cold")
+        p = Patient.objects.exclude(q1)
+        self.assertEqual(p.count(), 2)
+
+        # Flu
+        q1 = Q(eav__illness__icontains="flu")
+        p = Patient.objects.exclude(q1)
+        self.assertEqual(p.count(), 3)
+
+        # Empty
+        q1 = Q(eav__illness__isnull=False)
+        p = Patient.objects.filter(~q1)
+        self.assertEqual(p.count(), 1)
+
+
     def _order(self, ordering):
         query = Patient.objects.all().order_by(*ordering)
         return list(query.values_list('name', flat=True))